blending_generation_demo_colab.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  },
  "colab": {
   "name": "blending_generation_demo.ipynb",
   "provenance": [],
   "collapsed_sections": [],
   "toc_visible": true
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "560dbed5c45446528faed7456babd7d9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_view_name": "HBoxView",
      "_dom_classes": [],
      "_model_name": "HBoxModel",
      "_view_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_view_count": null,
      "_view_module_version": "1.5.0",
      "box_style": "",
      "layout": "IPY_MODEL_6d58a12c602548c287b225ce8c8763a5",
      "_model_module": "@jupyter-widgets/controls",
      "children": [
       "IPY_MODEL_612e34ecf6ca4ac495b83dc784364c3b",
       "IPY_MODEL_a2ec8e7efa8045f5af1c813f6cc24388"
      ]
     }
    },
    "6d58a12c602548c287b225ce8c8763a5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_view_name": "LayoutView",
      "grid_template_rows": null,
      "right": null,
      "justify_content": null,
      "_view_module": "@jupyter-widgets/base",
      "overflow": null,
      "_model_module_version": "1.2.0",
      "_view_count": null,
      "flex_flow": null,
      "width": null,
      "min_width": null,
      "border": null,
      "align_items": null,
      "bottom": null,
      "_model_module": "@jupyter-widgets/base",
      "top": null,
      "grid_column": null,
      "overflow_y": null,
      "overflow_x": null,
      "grid_auto_flow": null,
      "grid_area": null,
      "grid_template_columns": null,
      "flex": null,
      "_model_name": "LayoutModel",
      "justify_items": null,
      "grid_row": null,
      "max_height": null,
      "align_content": null,
      "visibility": null,
      "align_self": null,
      "height": null,
      "min_height": null,
      "padding": null,
      "grid_auto_rows": null,
      "grid_gap": null,
      "max_width": null,
      "order": null,
      "_view_module_version": "1.2.0",
      "grid_template_areas": null,
      "object_position": null,
      "object_fit": null,
      "grid_auto_columns": null,
      "margin": null,
      "display": null,
      "left": null
     }
    },
    "612e34ecf6ca4ac495b83dc784364c3b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_view_name": "ProgressView",
      "style": "IPY_MODEL_cce58498f29046c39b39b93aa5f76bdc",
      "_dom_classes": [],
      "description": "Downloading: 100%",
      "_model_name": "FloatProgressModel",
      "bar_style": "success",
      "max": 1042301,
      "_view_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "value": 1042301,
      "_view_count": null,
      "_view_module_version": "1.5.0",
      "orientation": "horizontal",
      "min": 0,
      "description_tooltip": null,
      "_model_module": "@jupyter-widgets/controls",
      "layout": "IPY_MODEL_1ed71d5ef4dc4f1e99093fb809ac8482"
     }
    },
    "a2ec8e7efa8045f5af1c813f6cc24388": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_view_name": "HTMLView",
      "style": "IPY_MODEL_41471c024acf445882a71f0c9d370e82",
      "_dom_classes": [],
      "description": "",
      "_model_name": "HTMLModel",
      "placeholder": "​",
      "_view_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "value": " 1.04M/1.04M [00:00&lt;00:00, 2.17MB/s]",
      "_view_count": null,
      "_view_module_version": "1.5.0",
      "description_tooltip": null,
      "_model_module": "@jupyter-widgets/controls",
      "layout": "IPY_MODEL_8bcdb851285b4be2aecff08e9924d2ee"
     }
    },
    "cce58498f29046c39b39b93aa5f76bdc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_view_name": "StyleView",
      "_model_name": "ProgressStyleModel",
      "description_width": "initial",
      "_view_module": "@jupyter-widgets/base",
      "_model_module_version": "1.5.0",
      "_view_count": null,
      "_view_module_version": "1.2.0",
      "bar_color": null,
      "_model_module": "@jupyter-widgets/controls"
     }
    },
    "1ed71d5ef4dc4f1e99093fb809ac8482": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_view_name": "LayoutView",
      "grid_template_rows": null,
      "right": null,
      "justify_content": null,
      "_view_module": "@jupyter-widgets/base",
      "overflow": null,
      "_model_module_version": "1.2.0",
      "_view_count": null,
      "flex_flow": null,
      "width": null,
      "min_width": null,
      "border": null,
      "align_items": null,
      "bottom": null,
      "_model_module": "@jupyter-widgets/base",
      "top": null,
      "grid_column": null,
      "overflow_y": null,
      "overflow_x": null,
      "grid_auto_flow": null,
      "grid_area": null,
      "grid_template_columns": null,
      "flex": null,
      "_model_name": "LayoutModel",
      "justify_items": null,
      "grid_row": null,
      "max_height": null,
      "align_content": null,
      "visibility": null,
      "align_self": null,
      "height": null,
      "min_height": null,
      "padding": null,
      "grid_auto_rows": null,
      "grid_gap": null,
      "max_width": null,
      "order": null,
      "_view_module_version": "1.2.0",
      "grid_template_areas": null,
      "object_position": null,
      "object_fit": null,
      "grid_auto_columns": null,
      "margin": null,
      "display": null,
      "left": null
     }
    },
    "41471c024acf445882a71f0c9d370e82": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_view_name": "StyleView",
      "_model_name": "DescriptionStyleModel",
      "description_width": "",
      "_view_module": "@jupyter-widgets/base",
      "_model_module_version": "1.5.0",
      "_view_count": null,
      "_view_module_version": "1.2.0",
      "_model_module": "@jupyter-widgets/controls"
     }
    },
    "8bcdb851285b4be2aecff08e9924d2ee": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_view_name": "LayoutView",
      "grid_template_rows": null,
      "right": null,
      "justify_content": null,
      "_view_module": "@jupyter-widgets/base",
      "overflow": null,
      "_model_module_version": "1.2.0",
      "_view_count": null,
      "flex_flow": null,
      "width": null,
      "min_width": null,
      "border": null,
      "align_items": null,
      "bottom": null,
      "_model_module": "@jupyter-widgets/base",
      "top": null,
      "grid_column": null,
      "overflow_y": null,
      "overflow_x": null,
      "grid_auto_flow": null,
      "grid_area": null,
      "grid_template_columns": null,
      "flex": null,
      "_model_name": "LayoutModel",
      "justify_items": null,
      "grid_row": null,
      "max_height": null,
      "align_content": null,
      "visibility": null,
      "align_self": null,
      "height": null,
      "min_height": null,
      "padding": null,
      "grid_auto_rows": null,
      "grid_gap": null,
      "max_width": null,
      "order": null,
      "_view_module_version": "1.2.0",
      "grid_template_areas": null,
      "object_position": null,
      "object_fit": null,
      "grid_auto_columns": null,
      "margin": null,
      "display": null,
      "left": null
     }
    },
    "56931d5ea9ef43bcab0156ca85c19404": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_view_name": "HBoxView",
      "_dom_classes": [],
      "_model_name": "HBoxModel",
      "_view_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_view_count": null,
      "_view_module_version": "1.5.0",
      "box_style": "",
      "layout": "IPY_MODEL_69d806c800a74d7f8f98bb644b4cf45a",
      "_model_module": "@jupyter-widgets/controls",
      "children": [
       "IPY_MODEL_1f8877b4cd964f37811869ab03550b2f",
       "IPY_MODEL_8b705612930346c9ba581f25260dce0e"
      ]
     }
    },
    "69d806c800a74d7f8f98bb644b4cf45a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_view_name": "LayoutView",
      "grid_template_rows": null,
      "right": null,
      "justify_content": null,
      "_view_module": "@jupyter-widgets/base",
      "overflow": null,
      "_model_module_version": "1.2.0",
      "_view_count": null,
      "flex_flow": null,
      "width": null,
      "min_width": null,
      "border": null,
      "align_items": null,
      "bottom": null,
      "_model_module": "@jupyter-widgets/base",
      "top": null,
      "grid_column": null,
      "overflow_y": null,
      "overflow_x": null,
      "grid_auto_flow": null,
      "grid_area": null,
      "grid_template_columns": null,
      "flex": null,
      "_model_name": "LayoutModel",
      "justify_items": null,
      "grid_row": null,
      "max_height": null,
      "align_content": null,
      "visibility": null,
      "align_self": null,
      "height": null,
      "min_height": null,
      "padding": null,
      "grid_auto_rows": null,
      "grid_gap": null,
      "max_width": null,
      "order": null,
      "_view_module_version": "1.2.0",
      "grid_template_areas": null,
      "object_position": null,
      "object_fit": null,
      "grid_auto_columns": null,
      "margin": null,
      "display": null,
      "left": null
     }
    },
    "1f8877b4cd964f37811869ab03550b2f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_view_name": "ProgressView",
      "style": "IPY_MODEL_c3296118f0084de591b5b87f892b8f51",
      "_dom_classes": [],
      "description": "Downloading: 100%",
      "_model_name": "FloatProgressModel",
      "bar_style": "success",
      "max": 456318,
      "_view_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "value": 456318,
      "_view_count": null,
      "_view_module_version": "1.5.0",
      "orientation": "horizontal",
      "min": 0,
      "description_tooltip": null,
      "_model_module": "@jupyter-widgets/controls",
      "layout": "IPY_MODEL_46975e449faf4b698ea6a37879518b7f"
     }
    },
    "8b705612930346c9ba581f25260dce0e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_view_name": "HTMLView",
      "style": "IPY_MODEL_5d392b5618d849cc88b40590709a21c2",
      "_dom_classes": [],
      "description": "",
      "_model_name": "HTMLModel",
      "placeholder": "​",
      "_view_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "value": " 456k/456k [00:00&lt;00:00, 1.47MB/s]",
      "_view_count": null,
      "_view_module_version": "1.5.0",
      "description_tooltip": null,
      "_model_module": "@jupyter-widgets/controls",
      "layout": "IPY_MODEL_28108107dd6048d39d6f79c1526eb809"
     }
    },
    "c3296118f0084de591b5b87f892b8f51": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_view_name": "StyleView",
      "_model_name": "ProgressStyleModel",
      "description_width": "initial",
      "_view_module": "@jupyter-widgets/base",
      "_model_module_version": "1.5.0",
      "_view_count": null,
      "_view_module_version": "1.2.0",
      "bar_color": null,
      "_model_module": "@jupyter-widgets/controls"
     }
    },
    "46975e449faf4b698ea6a37879518b7f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_view_name": "LayoutView",
      "grid_template_rows": null,
      "right": null,
      "justify_content": null,
      "_view_module": "@jupyter-widgets/base",
      "overflow": null,
      "_model_module_version": "1.2.0",
      "_view_count": null,
      "flex_flow": null,
      "width": null,
      "min_width": null,
      "border": null,
      "align_items": null,
      "bottom": null,
      "_model_module": "@jupyter-widgets/base",
      "top": null,
      "grid_column": null,
      "overflow_y": null,
      "overflow_x": null,
      "grid_auto_flow": null,
      "grid_area": null,
      "grid_template_columns": null,
      "flex": null,
      "_model_name": "LayoutModel",
      "justify_items": null,
      "grid_row": null,
      "max_height": null,
      "align_content": null,
      "visibility": null,
      "align_self": null,
      "height": null,
      "min_height": null,
      "padding": null,
      "grid_auto_rows": null,
      "grid_gap": null,
      "max_width": null,
      "order": null,
      "_view_module_version": "1.2.0",
      "grid_template_areas": null,
      "object_position": null,
      "object_fit": null,
      "grid_auto_columns": null,
      "margin": null,
      "display": null,
      "left": null
     }
    },
    "5d392b5618d849cc88b40590709a21c2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_view_name": "StyleView",
      "_model_name": "DescriptionStyleModel",
      "description_width": "",
      "_view_module": "@jupyter-widgets/base",
      "_model_module_version": "1.5.0",
      "_view_count": null,
      "_view_module_version": "1.2.0",
      "_model_module": "@jupyter-widgets/controls"
     }
    },
    "28108107dd6048d39d6f79c1526eb809": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_view_name": "LayoutView",
      "grid_template_rows": null,
      "right": null,
      "justify_content": null,
      "_view_module": "@jupyter-widgets/base",
      "overflow": null,
      "_model_module_version": "1.2.0",
      "_view_count": null,
      "flex_flow": null,
      "width": null,
      "min_width": null,
      "border": null,
      "align_items": null,
      "bottom": null,
      "_model_module": "@jupyter-widgets/base",
      "top": null,
      "grid_column": null,
      "overflow_y": null,
      "overflow_x": null,
      "grid_auto_flow": null,
      "grid_area": null,
      "grid_template_columns": null,
      "flex": null,
      "_model_name": "LayoutModel",
      "justify_items": null,
      "grid_row": null,
      "max_height": null,
      "align_content": null,
      "visibility": null,
      "align_self": null,
      "height": null,
      "min_height": null,
      "padding": null,
      "grid_auto_rows": null,
      "grid_gap": null,
      "max_width": null,
      "order": null,
      "_view_module_version": "1.2.0",
      "grid_template_areas": null,
      "object_position": null,
      "object_fit": null,
      "grid_auto_columns": null,
      "margin": null,
      "display": null,
      "left": null
     }
    }
   }
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "WGn0Z2WoOhCf"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/xxbidiao/plug-and-blend/blob/main/blending_generation_demo_colab.ipynb)\n",
    "\n",
    "## Introduction\n",
    "Plug-and-blend generate stories based on both a prompt and one or multiple continuously weighted topics.\n",
    "\n",
    "Here we show off some capabilities of Plug-and-blend, to illustrate its generation potential and how it can be used in an interactive setting."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "5MOOUmnBOhC2"
   },
   "source": [
    "## Setup\n",
    "Let's have code and dataset downloaded and set up."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fi052dDwRFEK",
    "outputId": "07d79cc9-e803-45e0-ffd8-414c57bbb01c"
   },
   "source": [
    "# Downloading the GeDi modifier model.\n",
    "!wget https://storage.googleapis.com/sfr-gedi-data/gedi_topic.zip\n",
    "import zipfile\n",
    "with zipfile.ZipFile('gedi_topic.zip', 'r') as zip_ref:\n",
    "    zip_ref.extractall('./')"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "--2021-03-31 02:01:03--  https://storage.googleapis.com/sfr-gedi-data/gedi_topic.zip\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.74.128, 209.85.145.128, 142.250.125.128, ...\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.74.128|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1318630072 (1.2G) [application/zip]\n",
      "Saving to: ‘gedi_topic.zip’\n",
      "\n",
      "gedi_topic.zip      100%[===================>]   1.23G   126MB/s    in 10s     \n",
      "\n",
      "2021-03-31 02:01:14 (125 MB/s) - ‘gedi_topic.zip’ saved [1318630072/1318630072]\n",
      "\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lKIarGzn4MbJ"
   },
   "source": [
    "Then we download the code archive and the base LM.\n",
    "\n",
    "Here we provide our fine-tuned GPT2-large on ROCStories dataset. You can use a different dataset to fine-tune your own model; as long as its tokenization is compatible with gpt-2, it should work."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "tVQRNhsZOnOA",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "c90927f2-0852-46fa-c650-136476d350f1"
   },
   "source": [
    "!gdown --id 1mkNr7unvQKBWayTZPSFlM7XhVMN8iNxA\n",
    "!unzip plug_and_blend_r1.zip\n",
    "\n",
    "!gdown --id 1Bhgfp2rZoCfU5tDPxZr5LV36WUfJXNOL\n",
    "!unzip rocstories_gpt2_large.zip -d baselm"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Downloading...\n",
      "From: https://drive.google.com/uc?id=1mkNr7unvQKBWayTZPSFlM7XhVMN8iNxA\n",
      "To: /content/plug_and_blend_r1.zip\n",
      "\r  0% 0.00/35.4k [00:00<?, ?B/s]\r100% 35.4k/35.4k [00:00<00:00, 58.1MB/s]\n",
      "Archive:  plug_and_blend_r1.zip\n",
      "   creating: gedi_helpers/\n",
      "  inflating: gedi_helpers/modeling_gpt2.py  \n",
      "  inflating: gedi_helpers/modeling_utils.py  \n",
      "  inflating: gedi_skill.py           \n",
      "  inflating: gedi_story_gen.py       \n",
      "Downloading...\n",
      "From: https://drive.google.com/uc?id=1Bhgfp2rZoCfU5tDPxZr5LV36WUfJXNOL\n",
      "To: /content/rocstories_gpt2_large.zip\n",
      "463MB [00:02, 206MB/s]\n",
      "Archive:  rocstories_gpt2_large.zip\n",
      "  inflating: baselm/config.json      \n",
      "  inflating: baselm/pytorch_model.bin  \n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iD9r9repQJQT"
   },
   "source": [
    "Finally, install these dependencies (colab may ask you to restart runtime since we use an older version of `torch`."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qfJtEWCmQH98",
    "outputId": "64a3dd1e-6275-4ad8-a932-5744c4186e56"
   },
   "source": [
    "!pip install transformers==3.5.1\n",
    "!pip install torch==1.4.0\n",
    "import nltk\n",
    "nltk.download('punkt')"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Collecting transformers==3.5.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3a/83/e74092e7f24a08d751aa59b37a9fc572b2e4af3918cb66f7766c3affb1b4/transformers-3.5.1-py3-none-any.whl (1.3MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3MB 5.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (3.0.12)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (4.41.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (2019.12.20)\n",
      "Collecting sentencepiece==0.1.91\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f2/e2/813dff3d72df2f49554204e7e5f73a3dc0f0eb1e3958a4cad3ef3fb278b7/sentencepiece-0.1.91-cp37-cp37m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 20.4MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 18.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (2.23.0)\n",
      "Collecting tokenizers==0.9.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7b/ac/f5ba028f0f097d855e1541301e946d4672eb0f30b6e25cb2369075f916d2/tokenizers-0.9.3-cp37-cp37m-manylinux1_x86_64.whl (2.9MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 32.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (1.19.5)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (20.9)\n",
      "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from transformers==3.5.1) (3.12.4)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.5.1) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.5.1) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.5.1) (1.0.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5.1) (2020.12.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5.1) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5.1) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.5.1) (2.10)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==3.5.1) (2.4.7)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf->transformers==3.5.1) (54.2.0)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=763a3f2fb312cac938789167c4d5af88e7bac9de27cbc980610631c8d29d56fe\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sentencepiece, sacremoses, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.9.3 transformers-3.5.1\n",
      "Collecting torch==1.4.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/3b/fa92ece1e58a6a48ec598bab327f39d69808133e5b2fb33002ca754e381e/torch-1.4.0-cp37-cp37m-manylinux1_x86_64.whl (753.4MB)\n",
      "\u001b[K     |████████████████████████████████| 753.4MB 21kB/s \n",
      "\u001b[31mERROR: torchvision 0.9.1+cu101 has requirement torch==1.8.1, but you'll have torch 1.4.0 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: torchtext 0.9.1 has requirement torch==1.8.1, but you'll have torch 1.4.0 which is incompatible.\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: torch\n",
      "  Found existing installation: torch 1.8.1+cu101\n",
      "    Uninstalling torch-1.8.1+cu101:\n",
      "      Successfully uninstalled torch-1.8.1+cu101\n",
      "Successfully installed torch-1.4.0\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 3
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "xVj7KGEwOhC4"
   },
   "source": [
    "## Prepare the models"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    },
    "id": "Pj8X-nSDOhC5"
   },
   "source": [
    "from gedi_skill import GediSkill"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "QzUaGT5QOhC6"
   },
   "source": [
    "## Load base model\n",
    "In this notebook, we have prepared all models and their paths set up for you.\n",
    "However, you can also bring your own base LM. \n",
    "For this demo, any model that uses the same vocabulary of GPT will work.\n",
    "\n",
    "If `base_model_path` is not `None`, it is treated as the path to base model.\n",
    "Otherwise, the original `gpt2-large` model is used instead."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    },
    "id": "k6WVstg4OhC8"
   },
   "source": [
    "base_model_path = \"baselm/\"\n",
    "gedi_path = \"gedi_topic/\"\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "-kzFyy4AOhC8"
   },
   "source": [
    "## Demo 1 - Let's generate some sentence!\n",
    "In this demo, we demonstrate individual-sentence blending generation capability of our blending generation language model.\n",
    "\n",
    "First, we load in the models (This may take some time):"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WY1TmiEY45T3",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 132,
     "referenced_widgets": [
      "560dbed5c45446528faed7456babd7d9",
      "6d58a12c602548c287b225ce8c8763a5",
      "612e34ecf6ca4ac495b83dc784364c3b",
      "a2ec8e7efa8045f5af1c813f6cc24388",
      "cce58498f29046c39b39b93aa5f76bdc",
      "1ed71d5ef4dc4f1e99093fb809ac8482",
      "41471c024acf445882a71f0c9d370e82",
      "8bcdb851285b4be2aecff08e9924d2ee",
      "56931d5ea9ef43bcab0156ca85c19404",
      "69d806c800a74d7f8f98bb644b4cf45a",
      "1f8877b4cd964f37811869ab03550b2f",
      "8b705612930346c9ba581f25260dce0e",
      "c3296118f0084de591b5b87f892b8f51",
      "46975e449faf4b698ea6a37879518b7f",
      "5d392b5618d849cc88b40590709a21c2",
      "28108107dd6048d39d6f79c1526eb809"
     ]
    },
    "outputId": "deb824a3-9a96-4a2f-a802-3083ebf13d4f"
   },
   "source": [
    "gedi_skill_obj = GediSkill(base_model_path,gedi_path)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "560dbed5c45446528faed7456babd7d9",
       "version_minor": 0,
       "version_major": 2
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1042301.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {
      "tags": []
     }
    },
    {
     "output_type": "stream",
     "text": [
      "\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56931d5ea9ef43bcab0156ca85c19404",
       "version_minor": 0,
       "version_major": 2
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     }
    },
    {
     "output_type": "stream",
     "text": [
      "\n",
      "no logit scale initialized for gpt2\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    },
    "id": "O9BiOW8zOhC9",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "9116d307-20f5-4331-e306-67527c5ab35c"
   },
   "source": [
    "# Then we set the parameters needed to generate the sentence.\n",
    "# Here we first provide a prompt...\n",
    "sentence = \"Welcome to Plug and Blend!\"\n",
    "\n",
    "# Then provide control codes. They can be arbitrary one-token words, so try something else! `Sports`,`Science`,`Business`,`World` works the best, but zero-shot topics are supported too.\n",
    "topic = {\"Science\":0.5,\"Business\":0.5}\n",
    "\n",
    "# Finally we set the control strength here. A higher value than 1 will result in more steering towards topic specified, but potentially less fluency (Especially when sentence is short and control strength is very high).\n",
    "# For example, setting it to 2 means 2x control strength.\n",
    "control_strength = 1\n",
    "\n",
    "# For ease of demo, scale it to internal strength, so that 1 in control_strength now means 30, the baseline value.\n",
    "control_strength = control_strength * gedi_skill_obj.disc_weight\n",
    "\n",
    "# Now just wait for the sentence to be generated.\n",
    "text = gedi_skill_obj.generate_one_sentence(sentence=sentence, topic=topic,extra_args = {\"disc_weight\":control_strength})\n",
    "print(text)\n",
    "print(\"OK!\")"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      " We're excited to share our passion for making delicious, healthy food.\n",
      "OK!\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    },
    "id": "8PFpN35aOhC-"
   },
   "source": [
    "## Demo 2: Run the story generation experience\n",
    "\n",
    "In this demo, you will provide Control Sketches (described in the paper) so that the planner will generate a 10-sentence story for you.\n",
    "\n",
    "An agent will interactively ask you (1) a 0-start topic index (e.g. 0 for `Sports` as in the default,  `['Sports','Science']` ), (2) a start point of the sketch, which specifies where the effect should emerge, in terms of sentence index (from 0 to 9), (3) an end point of the sketch. See the previous-run log for examples.\n",
    "\n",
    "(Colab may report out of memory if this demo is performed after demo 1. Feel free to restart the Colab instance.)\n",
    "\n",
    "For `topics` you can choose from `Sports`,`Science`,`Business`,`World`, or any other word that can be tokenized into one token (extra tokens will be ignored.)\n",
    "There can be more than 2 topics provided in `topics`. Try having more, and have fun!"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "pycharm": {
     "name": "#%% \n",
     "is_executing": false
    },
    "id": "13twwaJzOhC_",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "c678ffaa-e2e7-40a3-f764-1b4bc49862da"
   },
   "source": [
    "from story_gen import run_demo\n",
    "run_demo(\n",
    "    base_language_model_path = base_model_path, # use None to use original GPT2 model.\n",
    "    gedi_path=gedi_path,\n",
    "    topics=['Sports','Science']\n",
    ")\n",
    "\n",
    "\n"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Local location variables not used.\n",
      "Topics available: ['Sports', 'Science'] (Configure it in code)\n",
      "no logit scale initialized for gpt2\n",
      "Starting a new sketch. Input index of topic, or no input if no more new sketches:0\n",
      "This sketch is for topic: Sports\n",
      "Area to apply, start?0\n",
      "Area to apply, end?5\n",
      "Starting a new sketch. Input index of topic, or no input if no more new sketches:1\n",
      "This sketch is for topic: Science\n",
      "Area to apply, start?5\n",
      "Area to apply, end?10\n",
      "Starting a new sketch. Input index of topic, or no input if no more new sketches:\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "stream",
     "text": [
      "\r  0%|          | 0/10 [00:00<?, ?it/s]"
     ],
     "name": "stderr"
    },
    {
     "output_type": "stream",
     "text": [
      "Now generating...\n",
      "Planner output: [{'Sports': 1.0}, {'Sports': 1.0}, {'Sports': 0.9077361202459936, 'Science': 0.09226387975400638}, {'Sports': 0.8155224270282758, 'Science': 0.1844775729717242}, {'Sports': 0.6651435733326628, 'Science': 0.3348564266673373}, {'Sports': 0.4716058536173564, 'Science': 0.5283941463826437}, {'Sports': 0.2862435283632493, 'Science': 0.7137564716367507}, {'Sports': 0.15268457320416287, 'Science': 0.8473154267958372}, {'Sports': 0.0749034043928075, 'Science': 0.9250965956071925}, {'Science': 1.0}]\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [02:41<00:00, 16.15s/it]"
     ],
     "name": "stderr"
    },
    {
     "output_type": "stream",
     "text": [
      "[Sentence 0 is using Configuration {'Sports': 1.0}]\n",
      " Jackie Robinson was playing in the NBA.\n",
      "[Sentence 1 is using Configuration {'Sports': 1.0}]\n",
      " He had just played basketball for a few years.\n",
      "[Sentence 2 is using Configuration {'Sports': 0.9077361202459936, 'Science': 0.09226387975400638}]\n",
      "  He was looking forward to his first game of the season.\n",
      "[Sentence 3 is using Configuration {'Sports': 0.8155224270282758, 'Science': 0.1844775729717242}]\n",
      "  He decided to play with his friends and play against them in the court.\n",
      "[Sentence 4 is using Configuration {'Sports': 0.6651435733326628, 'Science': 0.3348564266673373}]\n",
      "     He got a lot of feedback from everyone who played against him, including some that were very excited about it!\n",
      "[Sentence 5 is using Configuration {'Sports': 0.4716058536173564, 'Science': 0.5283941463826437}]\n",
      " I was really happy when I saw how he played.\n",
      "[Sentence 6 is using Configuration {'Sports': 0.2862435283632493, 'Science': 0.7137564716367507}]\n",
      "  I also had to admit that my favorite player was the guy who beat me in the finals.\n",
      "[Sentence 7 is using Configuration {'Sports': 0.15268457320416287, 'Science': 0.8473154267958372}]\n",
      "  The computer game Super Mario 64 is a great game, but it's not perfect.\n",
      "[Sentence 8 is using Configuration {'Sports': 0.0749034043928075, 'Science': 0.9250965956071925}]\n",
      "  I played it on my laptop and found that I couldn't play it properly because of some bugs.\n",
      "[Sentence 9 is using Configuration {'Science': 1.0}]\n",
      "   The problem was that the graphics were bad, so I had to use an emulator instead of playing the game.\n",
      "-----\n",
      " Jackie Robinson was playing in the NBA.\n",
      " He had just played basketball for a few years.\n",
      "  He was looking forward to his first game of the season.\n",
      "  He decided to play with his friends and play against them in the court.\n",
      "     He got a lot of feedback from everyone who played against him, including some that were very excited about it!\n",
      " I was really happy when I saw how he played.\n",
      "  I also had to admit that my favorite player was the guy who beat me in the finals.\n",
      "  The computer game Super Mario 64 is a great game, but it's not perfect.\n",
      "  I played it on my laptop and found that I couldn't play it properly because of some bugs.\n",
      "   The problem was that the graphics were bad, so I had to use an emulator instead of playing the game.\n",
      "-----\n",
      "Done, thank you for using!\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "stream",
     "text": [
      "\n"
     ],
     "name": "stderr"
    }
   ]
  }
 ]
}