Text_classification_with_Bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text classification with Bert.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "19TXUiuROeguL-BpWH3W-WoZqsVwU3n9u",
      "authorship_tag": "ABX9TyOVnsKRCPbkff1D7xTS2bfG"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d5407737629344bd8dabfa98be7d94e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_54a8495e73a54adea001b7edc73b144d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6883de1adf9e409f84b323da9ef02501",
              "IPY_MODEL_ddf9f54162144b4e901109b4cff7eb26"
            ]
          }
        },
        "54a8495e73a54adea001b7edc73b144d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6883de1adf9e409f84b323da9ef02501": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d43f21e5c2d54af482522ed8584c062e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3be8323603624c32ae1c8a79cf027faf"
          }
        },
        "ddf9f54162144b4e901109b4cff7eb26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e07515b4023a4b888fe816ceb827e03b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [08:31&lt;00:00, 1.18s/B]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfa90b9459a74f94b82d6a9442303293"
          }
        },
        "d43f21e5c2d54af482522ed8584c062e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3be8323603624c32ae1c8a79cf027faf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e07515b4023a4b888fe816ceb827e03b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfa90b9459a74f94b82d6a9442303293": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b0a648bbd6a48b7a1be9f9073c40635": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0c63874fdabd4a2597960fe1504ddb8a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_78a22dda508b41ac830455c9dad09fb5",
              "IPY_MODEL_e2e9cd4bf3fc44bdbad6d3b9a5711251"
            ]
          }
        },
        "0c63874fdabd4a2597960fe1504ddb8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "78a22dda508b41ac830455c9dad09fb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e82884f1d39441579727fea9debeb83f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e263f7e1f879410cb04866727ab37783"
          }
        },
        "e2e9cd4bf3fc44bdbad6d3b9a5711251": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5bd4f6bca3574953bc7e668674a6eee8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [00:00&lt;00:00, 906kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8cd96e23e4fd48ee8bf13d13d8715cce"
          }
        },
        "e82884f1d39441579727fea9debeb83f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e263f7e1f879410cb04866727ab37783": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5bd4f6bca3574953bc7e668674a6eee8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8cd96e23e4fd48ee8bf13d13d8715cce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e780a5a4392143b1bc9e6608f90f998b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c730b7103e3f41e0bf642cc2fbc37e92",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1a05f0823eaa4330a52d1ac8178eb1b0",
              "IPY_MODEL_2e897009f62e4fc7a464bbdde3c2e85b"
            ]
          }
        },
        "c730b7103e3f41e0bf642cc2fbc37e92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a05f0823eaa4330a52d1ac8178eb1b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d4e817d86f294938b0af50c65c67fffb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435797,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435797,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a504d43f4504e459222e4df1d6c4a97"
          }
        },
        "2e897009f62e4fc7a464bbdde3c2e85b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0e3c70fb037b4f87aa1d92dba8756701",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436k/436k [08:30&lt;00:00, 853B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_daaf929e1d704c9c907f96a0141f10d5"
          }
        },
        "d4e817d86f294938b0af50c65c67fffb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a504d43f4504e459222e4df1d6c4a97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0e3c70fb037b4f87aa1d92dba8756701": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "daaf929e1d704c9c907f96a0141f10d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61ca2500f46a432184c9e1ae22b51f9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0112758a419942c1b246e676c7c998fe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0ae0322ec83e4495b9369d40fd216b59",
              "IPY_MODEL_794c2aec09d64ad4888a47324712db3c"
            ]
          }
        },
        "0112758a419942c1b246e676c7c998fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ae0322ec83e4495b9369d40fd216b59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c3e78fc3f5fd49bd8878d847e62b3d71",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8e51f9f4d1174934b01f27ff4078f2d5"
          }
        },
        "794c2aec09d64ad4888a47324712db3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_08a51f9b84034c16be1a5d1483b5ca0e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29.0/29.0 [00:00&lt;00:00, 78.1B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a27199e8fa34cfea96dfd49e5e457b5"
          }
        },
        "c3e78fc3f5fd49bd8878d847e62b3d71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8e51f9f4d1174934b01f27ff4078f2d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08a51f9b84034c16be1a5d1483b5ca0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a27199e8fa34cfea96dfd49e5e457b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c97f3ce3a6846678f88097756eea77d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_30284b217db242fbbef59ec1e8765c28",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ec96b374f4bd4c5daef1aeadd83aeb27",
              "IPY_MODEL_88506e1618274a74b9ea0ca0f053448d"
            ]
          }
        },
        "30284b217db242fbbef59ec1e8765c28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ec96b374f4bd4c5daef1aeadd83aeb27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b5b31b1f25b947b79b98b9229d894e95",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435779157,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435779157,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2e06820e832c41c1879e7043f74de9d9"
          }
        },
        "88506e1618274a74b9ea0ca0f053448d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_80072e19a8d94c509d9675a7985fceb8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436M/436M [08:30&lt;00:00, 854kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6365eb151826490693fe803c774c8785"
          }
        },
        "b5b31b1f25b947b79b98b9229d894e95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2e06820e832c41c1879e7043f74de9d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80072e19a8d94c509d9675a7985fceb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6365eb151826490693fe803c774c8785": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KZF3fsf7Axxo"
      },
      "source": [
        "# Text classification on movie reviews Dataset from stanford with Bert"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7XZxdcVoEHf7"
      },
      "source": [
        "fig = dict({\n",
        "    \"data\": [{\"type\": \"bar\",\n",
        "              \"x\": [1, 2, 3],\n",
        "              \"y\": [1, 3, 2]}],\n",
        "    \"layout\": {\"title\": {\"text\": \"A Figure Specified By Python Dictionary\"}}\n",
        "})\n",
        "\n",
        "# To display the figure defined by this dict, use the low-level plotly.io.show function\n",
        "import plotly.io as pio\n",
        "\n",
        "pio.show(fig)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JSe2NKx2iL6Q",
        "outputId": "579d2539-ecec-4ceb-f1c9-dd2456d91a5b"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/transformers_for_nlp\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SDSbKYlhhRo0"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_l5pRX5u1xDG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fcbf8e57-5e04-49fd-e10f-7ca5ebe62d14"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/transformers_for_nlp\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AVEQWnnC1KSE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0e8bde22-b423-48c9-e9a2-fb92c461aefe"
      },
      "source": [
        "!pip install transformers\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 5.3MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 16.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 45.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=aebcffdcc9b096bd0f9bce25864c284bb01f739ed1626927a57d7d7ce04aef06\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.44 tokenizers-0.10.2 transformers-4.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E7R_MsQ51Sbf"
      },
      "source": [
        "# Downloading the dataset.\n",
        "#!wget -q -nc http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHDlMAqd8ULs"
      },
      "source": [
        "# Unziping the dataset.\n",
        "#!tar -zxf /content/drive/MyDrive/transformers_for_nlp/aclImdb_v1.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9v3rD7pcAYDi",
        "outputId": "35dbac87-5991-453a-ded3-5fcae2ee7644"
      },
      "source": [
        "!pip install -q git+https://github.com/gmihaila/ml_things.git\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 71kB 3.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 10.3MB 6.4MB/s \n",
            "\u001b[?25h  Building wheel for ml-things (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SjftWj9fxn3U",
        "outputId": "d0b94305-edfb-444f-f050-e2bcfc8d5130"
      },
      "source": [
        "import os\n",
        "import io\n",
        "from tqdm.notebook import tqdm\n",
        "from torch.utils.data import Dataset , DataLoader \n",
        "from ml_things import plot_dict , fix_text , plot_confusion_matrix\n",
        "from sklearn.metrics import classification_report , accuracy_score\n",
        "from transformers import (AutoConfig ,AutoModelForSequenceClassification ,AutoTokenizer,AdamW,\n",
        "                          get_linear_schedule_with_warmup , set_seed)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/packaging/version.py:130: DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release\n",
            "  DeprecationWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Mg42ILdD0IU"
      },
      "source": [
        "import torch\n",
        "from torch import nn \n",
        "import torch.nn.functional as F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z63meho0a03C"
      },
      "source": [
        "set_seed(26)\n",
        "epochs = 4\n",
        "batches =  32 # as max seq length is small but for max _Seq_len =512 batch_Size should be very small (due togpu issues)\n",
        "max_seq_length = 50\n",
        "device =torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "if device == 'cpu' :\n",
        "    print(\"Using cpu ,This will b slow\")\n",
        "model_name = 'bert-base-cased'\n",
        "\n",
        "labels_d = { \"pos\" : 1 , \"neg\" : 0}\n",
        "n_labels = len(labels)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "13lSCB4qa44L"
      },
      "source": [
        "class MRDataSet(Dataset):\n",
        "    r\"\"\"\n",
        "    This class can be used as is as long\n",
        "    as the `dataloader` outputs a batch in dictionary format that can be passed \n",
        "    straight into the model - `model(**batch)`.\n",
        " \n",
        "  Arguments:\n",
        " \n",
        "    path (:obj:`str`):\n",
        "        Path to the data partition.\n",
        "     \n",
        "    tokenizer (:obj:`transformers.tokenization_?`):\n",
        "        Transformer type tokenizer used to process raw text into numbers.\n",
        " \n",
        "    labels (:obj:`dict`):\n",
        "        Dictionary to encode any labels names into numbers. Keys map to \n",
        "        labels names and Values map to number associated to those labels.\n",
        " \n",
        "    max_sequence_len (:obj:`int`, `optional`)\n",
        "        Value to indicate the maximum desired sequence to truncate or pad text\n",
        "        sequences. If no value is passed it will used maximum sequence size\n",
        "        supported by the tokenizer and model.\n",
        "     \"\"\"\n",
        "    def __init__(self , path = \"/content/drive/MyDrive/transformers_for_nlp/aclImdb/train\"  ,\\\n",
        "                tokenizer = None , \\\n",
        "                labels_d =labels_d, max_seq_len = max_seq_length):\n",
        "        super(MRDataSet , self).__init__()\n",
        "\n",
        "        if not os.path.isdir(path):\n",
        "             print(\"Path to directory does not exist.\")\n",
        "\n",
        "        texts = []\n",
        "        labels = []\n",
        "\n",
        "        #the labels are defined by folders with data we loop through each label.\n",
        "\n",
        "        for label , label_id in tqdm(labels_d.items()):\n",
        "             text_path  = os.path.join(path , label) #path/pos | path/neg\n",
        "\n",
        "             all_file_names = os.listdir(text_path)[:1000]\n",
        "             for file_name in tqdm(all_file_names):\n",
        "                 file_path = os.path.join(text_path , file_name)\n",
        "\n",
        "                 text = io.open(file_path , mode = 'r' , encoding = 'utf-8').read()\n",
        "                 text = fix_text(text) #solves any encoding issues\n",
        "                 texts.append(text)\n",
        "                 labels.append(label_id)\n",
        "        self.n = len(labels)\n",
        "        #using tokenizer on text ,returns a dictionary with tokenized integer ids under input_ids key\n",
        "        self.tokenized_inputs = tokenizer(texts ,add_special_tokens = True , truncation = True, padding = True ,\n",
        "                                          return_tensors = 'pt' , max_length = max_seq_len)\n",
        "        self.seq_len = self.tokenized_inputs['input_ids'].shape[-1] #length of sequence\n",
        "\n",
        "        self.tokenized_inputs.update({\"labels\" : torch.tensor(labels)})\n",
        "\n",
        "    def __len__(self) :\n",
        "            return self.n\n",
        "\n",
        "    def __getitem__(self , index):\n",
        "            r\"\"\"\n",
        "            Arguments:\n",
        " \n",
        "            index (:obj:`int`):\n",
        "                Index position to pick an example to return.\n",
        "        \n",
        "            Returns:\n",
        "            :obj:`Dict[str, object]`: Dictionary of inputs that feed into the model.\n",
        "            It holddes the statement `model(**Returned Dictionary)`.\n",
        "\n",
        "            \"\"\"\n",
        "            return {key : self.tokenized_inputs[key][items] for key in self.tokenized_inputs.keys()}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "oS-pbSv0aiKK",
        "outputId": "966eb647-4c64-4392-f5a3-93e6ef9c79b5"
      },
      "source": [
        "d = Dataset(\"/content/drive/MyDrive/transformers_for_nlp/aclImdb/train\" , AutoTokenizer , labels , max_seq_len)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-de87d271da11>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0md\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/content/drive/MyDrive/transformers_for_nlp/aclImdb/train\"\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mAutoTokenizer\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mmax_seq_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/lib/python3.7/typing.py\u001b[0m in \u001b[0;36m__new__\u001b[0;34m(cls, *args, **kwds)\u001b[0m\n\u001b[1;32m    819\u001b[0m             \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__new__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 821\u001b[0;31m             \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__new__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    822\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: object.__new__() takes exactly one argument (the type to instantiate)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EUoXz8hh3oE1"
      },
      "source": [
        "def train(dataloader , optimizer_ , scheduler_ , device_):\n",
        "    r\"\"\"\n",
        "\n",
        "    Performs a single training pass of the \" model \" which is a global variable loaded on device_\n",
        "\n",
        "    Arguments:\n",
        "\n",
        "    dataloader (:obj:`torch.utils.data.dataloader.DataLoader`):\n",
        "        Parsed data into batches of tensors.\n",
        "\n",
        "    optimizer_ (:obj:`transformers.optimization.AdamW`):\n",
        "        Optimizer used for training.\n",
        "\n",
        "    scheduler_ (:obj:`torch.optim.lr_scheduler.LambdaLR`):\n",
        "        PyTorch scheduler.\n",
        "\n",
        "    device_ (:obj:`torch.device`):\n",
        "        Device used to load tensors before feeding to model.\n",
        "\n",
        "    Returns:\n",
        "\n",
        "    :obj:`List[List[int], List[int], float]`: List of [True Labels, Predicted\n",
        "        Labels, Train Average Loss].\n",
        "    \"\"\"\n",
        "\n",
        "    global model\n",
        "\n",
        "    preds , true_labels = [] , [] \n",
        "    total_loss = 0\n",
        "    model.train()\n",
        "\n",
        "    for batch in tqdm(dataloader , total = len(dataloader)) :\n",
        "        #Adding to original labels - to be used later for evaluation\n",
        "        true_labels += batch[\"labels\"].numpy().flatten().tolist()\n",
        "        batch = {k:v.type(torch.long).to(device_) for k,v in batch.items()}\n",
        "\n",
        "        model.zero_grad()\n",
        "        #This will return the loss (rather than the model output) because we have provided the `labels`.\n",
        "        # model always returns a tuple \n",
        "        out = model(**batch)\n",
        "\n",
        "        loss , logits = out[:2]\n",
        "\n",
        "        total_loss += loss.item()\n",
        "        loss.backward()\n",
        "        #gradient clipping `\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)\n",
        "\n",
        "        optimizer_.step()\n",
        "        schedular_.step\n",
        "\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        # Convertion of these logits to list of predicted labels values.\n",
        "        preds += logits.argmax(axis = -1).flatten().tolist()\n",
        "\n",
        "    average_epoch_loss = total_loss / len(dataloader)\n",
        "\n",
        "    return true_labels,preds,average_epoch_loss\n",
        "\n",
        "\n",
        "\n",
        " "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ld8TcReBb1OG"
      },
      "source": [
        "def valiadation(dataloader , device_):\n",
        "    \"\"\"\n",
        "    function that performs validation on validation data\n",
        "     Arguments:\n",
        " \n",
        "    dataloader (:obj:`torch.utils.data.dataloader.DataLoader`):\n",
        "          Parsed data into batches of tensors.\n",
        " \n",
        "    device_ (:obj:`torch.device`):\n",
        "          Device used to load tensors before feeding to model.\n",
        " \n",
        "     Returns:\n",
        "     \n",
        "    :True Labels, Predictions,  Average Loss\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    global model \n",
        "    preds , true_labels= [] , []\n",
        "    total_loss = 0 \n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    for batch in tqdm(dataloader , total = len(dataloader)) :\n",
        "        #Adding to original labels - to be used later for evaluation\n",
        "        true_labels += batch[\"labels\"].numpy().flatten().tolist()\n",
        "        batch = {k:v.type(torch.long).to(device_) for k,v in batch.items()}\n",
        "\n",
        "        with torch.no_grad():\n",
        "            out = model(**batch)\n",
        "            loss , logits = out[:2]\n",
        "            logits = logits.detach().cpu().numpy()\n",
        "            total_loss += loss\n",
        "\n",
        "            preds += logits.argmax(axis = -1).flatten().tolist()\n",
        "\n",
        "    average_validation_loss = total_loss / len(dataloader)\n",
        "\n",
        "    return true_labels , preds , average_validation_loss\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371,
          "referenced_widgets": [
            "d5407737629344bd8dabfa98be7d94e4",
            "54a8495e73a54adea001b7edc73b144d",
            "6883de1adf9e409f84b323da9ef02501",
            "ddf9f54162144b4e901109b4cff7eb26",
            "d43f21e5c2d54af482522ed8584c062e",
            "3be8323603624c32ae1c8a79cf027faf",
            "e07515b4023a4b888fe816ceb827e03b",
            "cfa90b9459a74f94b82d6a9442303293",
            "7b0a648bbd6a48b7a1be9f9073c40635",
            "0c63874fdabd4a2597960fe1504ddb8a",
            "78a22dda508b41ac830455c9dad09fb5",
            "e2e9cd4bf3fc44bdbad6d3b9a5711251",
            "e82884f1d39441579727fea9debeb83f",
            "e263f7e1f879410cb04866727ab37783",
            "5bd4f6bca3574953bc7e668674a6eee8",
            "8cd96e23e4fd48ee8bf13d13d8715cce",
            "e780a5a4392143b1bc9e6608f90f998b",
            "c730b7103e3f41e0bf642cc2fbc37e92",
            "1a05f0823eaa4330a52d1ac8178eb1b0",
            "2e897009f62e4fc7a464bbdde3c2e85b",
            "d4e817d86f294938b0af50c65c67fffb",
            "3a504d43f4504e459222e4df1d6c4a97",
            "0e3c70fb037b4f87aa1d92dba8756701",
            "daaf929e1d704c9c907f96a0141f10d5",
            "61ca2500f46a432184c9e1ae22b51f9e",
            "0112758a419942c1b246e676c7c998fe",
            "0ae0322ec83e4495b9369d40fd216b59",
            "794c2aec09d64ad4888a47324712db3c",
            "c3e78fc3f5fd49bd8878d847e62b3d71",
            "8e51f9f4d1174934b01f27ff4078f2d5",
            "08a51f9b84034c16be1a5d1483b5ca0e",
            "6a27199e8fa34cfea96dfd49e5e457b5",
            "2c97f3ce3a6846678f88097756eea77d",
            "30284b217db242fbbef59ec1e8765c28",
            "ec96b374f4bd4c5daef1aeadd83aeb27",
            "88506e1618274a74b9ea0ca0f053448d",
            "b5b31b1f25b947b79b98b9229d894e95",
            "2e06820e832c41c1879e7043f74de9d9",
            "80072e19a8d94c509d9675a7985fceb8",
            "6365eb151826490693fe803c774c8785"
          ]
        },
        "id": "6b9KgRbDWtxs",
        "outputId": "2dc614b8-a26e-4f31-ba79-a16d2c56409a"
      },
      "source": [
        "model_config = AutoConfig.from_pretrained(pretrained_model_name_or_path=model_name , num_labels = n_labels)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path=model_name , config=model_config)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5407737629344bd8dabfa98be7d94e4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7b0a648bbd6a48b7a1be9f9073c40635",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e780a5a4392143b1bc9e6608f90f998b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435797.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61ca2500f46a432184c9e1ae22b51f9e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=29.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2c97f3ce3a6846678f88097756eea77d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435779157.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZDsRQJnkQQh"
      },
      "source": [
        "model.to(device)\n",
        "Tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_name)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGhhU6BFXSSd"
      },
      "source": [
        "\n",
        "training_dataset = MRDataSet( tokenizer = Tokenizer )\n",
        "\n",
        "\n",
        "traning_dataloader = DataLoader(training_dataset , batch_size = batch_size , shuffle =True)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6wpoo68YVya"
      },
      "source": [
        "validation_dataset = MRDataSet( tokenizer = Tokenizer)\n",
        "validation_dataloader = DataLoader(validation_dataset)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbMYGD8gAX-j"
      },
      "source": [
        "optimizer = AdamW(model.parameters()  ,lr = 2e-5  ,eps = 1e-8)\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "# `train_dataloader` contains batched data so `len(train_dataloader)` gives \n",
        "# us the number of batches.\n",
        "total_steps = len(train_dataloader) * epochs\n",
        " \n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)\n",
        " \n",
        "# Store the average loss after each epoch so we can plot them.\n",
        "all_loss = {'train_loss':[], 'val_loss':[]}\n",
        "all_acc = {'train_acc':[], 'val_acc':[]}\n",
        " \n",
        "# Loop through each epoch.\n",
        "print('Epoch')\n",
        "for epoch in tqdm(range(epochs)):\n",
        "  print()\n",
        "  print('Training on batches...')\n",
        "  # Perform one full pass over the training set.\n",
        "  train_labels, train_predict, train_loss = train(train_dataloader, optimizer, scheduler, device)\n",
        "  train_acc = accuracy_score(train_labels, train_predict)\n",
        " \n",
        "  # Get prediction form model on validation data. \n",
        "  print('Validation on batches...')\n",
        "  valid_labels, valid_predict, val_loss = validation(valid_dataloader, device)\n",
        "  val_acc = accuracy_score(valid_labels, valid_predict)\n",
        " \n",
        "  # Print loss and accuracy values to see how training evolves.\n",
        "  print(\"  train_loss: %.5f - val_loss: %.5f - train_acc: %.5f - valid_acc: %.5f\"%(train_loss, val_loss, train_acc, val_acc))\n",
        "  print()\n",
        " \n",
        "  # Store the loss value for plotting the learning curve.\n",
        "  all_loss['train_loss'].append(train_loss)\n",
        "  all_loss['val_loss'].append(val_loss)\n",
        "  all_acc['train_acc'].append(train_acc)\n",
        "  all_acc['val_acc'].append(val_acc)\n",
        " \n",
        "# Plot loss curves.\n",
        "plot_dict(all_loss, use_xlabel='Epochs', use_ylabel='Value', use_linestyles=['-', '--'])\n",
        " \n",
        "# Plot accuracy curves.\n",
        "plot_dict(all_acc, use_xlabel='Epochs', use_ylabel='Value', use_linestyles=['-', '--'])"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}