L5_Chat_with_any_LLM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c5753a0c",
   "metadata": {},
   "source": [
    "# L5: Chat with any LLM! 💬"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a01a3724",
   "metadata": {},
   "source": [
    "Load your HF API key and relevant Python libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fa6fa00-6bd1-4839-bcaf-8bae9267ee79",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import io\n",
    "import IPython.display\n",
    "from PIL import Image\n",
    "import base64 \n",
    "import requests \n",
    "requests.adapters.DEFAULT_TIMEOUT = 60\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv()) # read local .env file\n",
    "hf_api_key = os.environ['HF_API_KEY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "095da8fe-24aa-4dc7-8e08-aa2f949ae21f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "import requests, json\n",
    "from text_generation import Client\n",
    "\n",
    "#FalcomLM-instruct endpoint on the text_generation library\n",
    "client = Client(os.environ['HF_API_FALCOM_BASE'], headers={\"Authorization\": f\"Basic {hf_api_key}\"}, timeout=120)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfe6fc97",
   "metadata": {},
   "source": [
    "## Building an app to chat with any LLM!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "745a3c9b",
   "metadata": {},
   "source": [
    "Here we'll be using an [Inference Endpoint](https://huggingface.co/inference-endpoints) for `falcon-40b-instruct` , one of best ranking open source LLM on the [🤗 Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard). \n",
    "\n",
    "To run it locally, one can use the [Transformers library](https://huggingface.co/docs/transformers/index) or the [text-generation-inference](https://github.com/huggingface/text-generation-inference) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7065860-3c0b-490d-9e7c-22e5b79fc004",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"Has math been invented or discovered?\"\n",
    "client.generate(prompt, max_new_tokens=256).generated_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dcb659e-b71b-46da-b9d2-6ee62498995f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Back to Lesson 2, time flies!\n",
    "import gradio as gr\n",
    "def generate(input, slider):\n",
    "    output = client.generate(input, max_new_tokens=slider).generated_text\n",
    "    return output\n",
    "\n",
    "demo = gr.Interface(fn=generate, inputs=[gr.Textbox(label=\"Prompt\"), gr.Slider(label=\"Max new tokens\", value=20,  maximum=1024, minimum=1)], outputs=[gr.Textbox(label=\"Completion\")])\n",
    "gr.close_all()\n",
    "demo.launch(share=True, server_port=int(os.environ['PORT1']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e5f55e2",
   "metadata": {},
   "source": [
    "## `gr.Chatbot()` to the rescue!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43beebb7-40a6-4af5-a701-882821b6ed36",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def respond(message, chat_history):\n",
    "        #No LLM here, just respond with a random pre-made message\n",
    "        bot_message = random.choice([\"Tell me more about it\", \n",
    "                                     \"Cool, but I'm not interested\", \n",
    "                                     \"Hmmmm, ok then\"]) \n",
    "        chat_history.append((message, bot_message))\n",
    "        return \"\", chat_history\n",
    "\n",
    "with gr.Blocks() as demo:\n",
    "    chatbot = gr.Chatbot(height=240) #just to fit the notebook\n",
    "    msg = gr.Textbox(label=\"Prompt\")\n",
    "    btn = gr.Button(\"Submit\")\n",
    "    clear = gr.ClearButton(components=[msg, chatbot], value=\"Clear console\")\n",
    "\n",
    "    btn.click(respond, inputs=[msg, chatbot], outputs=[msg, chatbot])\n",
    "    msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot]) #Press enter to submit\n",
    "gr.close_all()\n",
    "demo.launch(share=True, server_port=int(os.environ['PORT2']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55bae99d-7a63-4a40-bab7-de7d10b8ab1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_chat_prompt(message, chat_history):\n",
    "    prompt = \"\"\n",
    "    for turn in chat_history:\n",
    "        user_message, bot_message = turn\n",
    "        prompt = f\"{prompt}\\nUser: {user_message}\\nAssistant: {bot_message}\"\n",
    "    prompt = f\"{prompt}\\nUser: {message}\\nAssistant:\"\n",
    "    return prompt\n",
    "\n",
    "def respond(message, chat_history):\n",
    "        formatted_prompt = format_chat_prompt(message, chat_history)\n",
    "        bot_message = client.generate(formatted_prompt,\n",
    "                                     max_new_tokens=1024,\n",
    "                                     stop_sequences=[\"\\nUser:\", \"<|endoftext|>\"]).generated_text\n",
    "        chat_history.append((message, bot_message))\n",
    "        return \"\", chat_history\n",
    "\n",
    "with gr.Blocks() as demo:\n",
    "    chatbot = gr.Chatbot(height=240) #just to fit the notebook\n",
    "    msg = gr.Textbox(label=\"Prompt\")\n",
    "    btn = gr.Button(\"Submit\")\n",
    "    clear = gr.ClearButton(components=[msg, chatbot], value=\"Clear console\")\n",
    "\n",
    "    btn.click(respond, inputs=[msg, chatbot], outputs=[msg, chatbot])\n",
    "    msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot]) #Press enter to submit\n",
    "gr.close_all()\n",
    "demo.launch(share=True, server_port=int(os.environ['PORT3']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f22b8de8",
   "metadata": {},
   "source": [
    "### Adding other advanced features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09873dfd-5b6c-41d6-9479-12e8c8894295",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_chat_prompt(message, chat_history, instruction):\n",
    "    prompt = f\"System:{instruction}\"\n",
    "    for turn in chat_history:\n",
    "        user_message, bot_message = turn\n",
    "        prompt = f\"{prompt}\\nUser: {user_message}\\nAssistant: {bot_message}\"\n",
    "    prompt = f\"{prompt}\\nUser: {message}\\nAssistant:\"\n",
    "    return prompt\n",
    "\n",
    "def respond(message, chat_history, instruction, temperature=0.7):\n",
    "    prompt = format_chat_prompt(message, chat_history, instruction)\n",
    "    chat_history = chat_history + [[message, \"\"]]\n",
    "    stream = client.generate_stream(prompt,\n",
    "                                      max_new_tokens=1024,\n",
    "                                      stop_sequences=[\"\\nUser:\", \"<|endoftext|>\"],\n",
    "                                      temperature=temperature)\n",
    "                                      #stop_sequences to not generate the user answer\n",
    "    acc_text = \"\"\n",
    "    #Streaming the tokens\n",
    "    for idx, response in enumerate(stream):\n",
    "            text_token = response.token.text\n",
    "\n",
    "            if response.details:\n",
    "                return\n",
    "\n",
    "            if idx == 0 and text_token.startswith(\" \"):\n",
    "                text_token = text_token[1:]\n",
    "\n",
    "            acc_text += text_token\n",
    "            last_turn = list(chat_history.pop(-1))\n",
    "            last_turn[-1] += acc_text\n",
    "            chat_history = chat_history + [last_turn]\n",
    "            yield \"\", chat_history\n",
    "            acc_text = \"\"\n",
    "\n",
    "with gr.Blocks() as demo:\n",
    "    chatbot = gr.Chatbot(height=240) #just to fit the notebook\n",
    "    msg = gr.Textbox(label=\"Prompt\")\n",
    "    with gr.Accordion(label=\"Advanced options\",open=False):\n",
    "        system = gr.Textbox(label=\"System message\", lines=2, value=\"A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.\")\n",
    "        temperature = gr.Slider(label=\"temperature\", minimum=0.1, maximum=1, value=0.7, step=0.1)\n",
    "    btn = gr.Button(\"Submit\")\n",
    "    clear = gr.ClearButton(components=[msg, chatbot], value=\"Clear console\")\n",
    "\n",
    "    btn.click(respond, inputs=[msg, chatbot, system], outputs=[msg, chatbot])\n",
    "    msg.submit(respond, inputs=[msg, chatbot, system], outputs=[msg, chatbot]) #Press enter to submit\n",
    "gr.close_all()\n",
    "demo.queue().launch(share=True, server_port=int(os.environ['PORT4']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d9ec80a-39ad-4f58-b79e-4f413c5074c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "gr.close_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c7e28b0-0300-4e58-a01d-1d7eabe188eb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ca70638-2db0-4a01-9b6c-b866ab50b891",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9571be90-43c3-4663-ae8a-6ab54e634161",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9beb304e-fdb6-484b-a04a-f352f6d70ec3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b666122e-95f3-42ac-8208-75b43e20cfc6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b39aca63-841b-47f1-972c-5097231cc40e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "792baf87-7f2e-4ed3-a9a2-3c86a53a3a37",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea585333-a583-4b92-91d1-ea6ee7e1aa0c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77afcf4d-25d4-4c89-a6ba-c222cb476f2e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f8fbdcb-b583-4645-baa3-397c8d06cdc3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3c64ed0-9dc6-4740-92b2-8bfef66ae258",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1d10d21-6a67-48bc-94fd-17a9d6c645d4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ff096e7-8e2e-4791-a79b-9c5ef1c45a21",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62c34731-b8f9-4dcd-888a-f5f5f5cc9eaf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2683f4f8-ab96-4685-adfa-11e872a5f373",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e72fcaf-3740-4704-b299-d8096d79b99c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dee4edd3-1b95-468a-8177-12aa0677b92a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8104dd16-9c65-42a3-bf09-504aa67c1c10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12cf9b3a-4202-4e3a-9c6b-941fa1290ab8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}