3_Advanced_Usage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7dfed66f-9100-478b-a67f-493daca1fe68",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !cd text-generation-webui && python server.py --model-menu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "77d1e871-71bb-419e-8293-dd90d50f7d5b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting server...\n",
      "INFO:\u001b[32mLoading TheBloke_vicuna-7B-1.1-GPTQ-4bit-128g...\u001b[0m\n",
      "CUDA extension not installed.\n",
      "INFO:\u001b[32mFound the following quantized model: models/TheBloke_vicuna-7B-1.1-GPTQ-4bit-128g/vicuna-7B-1.1-GPTQ-4bit-128g.safetensors\u001b[0m\n",
      "INFO:\u001b[32mLoaded the model in 2.90 seconds.\n",
      "\u001b[0m\n",
      "Starting streaming server at ws://127.0.0.1:6667/api/v1/stream\n",
      "Starting API at http://127.0.0.1:6666/api\n",
      "Server started!\n",
      "127.0.0.1 - - [05/Jun/2023 00:51:37] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Running on local URL:  http://127.0.0.1:6668\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n",
      "Output generated in 22.05 seconds (2.54 tokens/s, 56 tokens, context 244, seed 0)\n",
      "127.0.0.1 - - [05/Jun/2023 00:52:00] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 113.19 seconds (2.60 tokens/s, 294 tokens, context 248, seed 1)\n",
      "127.0.0.1 - - [05/Jun/2023 00:53:53] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 29.99 seconds (2.60 tokens/s, 78 tokens, context 204, seed 2)\n",
      "127.0.0.1 - - [05/Jun/2023 00:54:23] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 16.93 seconds (2.60 tokens/s, 44 tokens, context 237, seed 3)\n",
      "127.0.0.1 - - [05/Jun/2023 00:54:40] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 49.77 seconds (2.59 tokens/s, 129 tokens, context 240, seed 4)\n",
      "127.0.0.1 - - [05/Jun/2023 00:55:30] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 42.37 seconds (2.60 tokens/s, 110 tokens, context 215, seed 5)\n",
      "127.0.0.1 - - [05/Jun/2023 00:56:13] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 20.01 seconds (2.60 tokens/s, 52 tokens, context 239, seed 6)\n",
      "127.0.0.1 - - [05/Jun/2023 00:56:33] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 12.32 seconds (2.60 tokens/s, 32 tokens, context 215, seed 7)\n",
      "127.0.0.1 - - [05/Jun/2023 00:56:45] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 29.22 seconds (2.60 tokens/s, 76 tokens, context 235, seed 8)\n",
      "127.0.0.1 - - [05/Jun/2023 00:57:14] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 41.18 seconds (2.60 tokens/s, 107 tokens, context 216, seed 9)\n",
      "127.0.0.1 - - [05/Jun/2023 00:57:56] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 164.00 seconds (2.59 tokens/s, 425 tokens, context 289, seed 10)\n",
      "127.0.0.1 - - [05/Jun/2023 01:00:40] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 29.64 seconds (2.60 tokens/s, 77 tokens, context 200, seed 11)\n",
      "127.0.0.1 - - [05/Jun/2023 01:01:10] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 43.85 seconds (2.60 tokens/s, 114 tokens, context 217, seed 12)\n",
      "127.0.0.1 - - [05/Jun/2023 01:01:54] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 24.22 seconds (2.60 tokens/s, 63 tokens, context 194, seed 13)\n",
      "127.0.0.1 - - [05/Jun/2023 01:02:18] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 14.99 seconds (2.60 tokens/s, 39 tokens, context 175, seed 14)\n",
      "127.0.0.1 - - [05/Jun/2023 01:02:33] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 12.31 seconds (2.60 tokens/s, 32 tokens, context 188, seed 15)\n",
      "127.0.0.1 - - [05/Jun/2023 01:02:46] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Output generated in 18.45 seconds (2.60 tokens/s, 48 tokens, context 195, seed 16)\n",
      "127.0.0.1 - - [05/Jun/2023 01:03:04] \"POST /api/v1/generate HTTP/1.1\" 200 -\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/ec2-user/SageMaker/text-generation-webui/server.py\", line 1118, in <module>\n",
      "    time.sleep(0.5)\n",
      "KeyboardInterrupt\n",
      "terminate called without an active exception\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_1862/98509441.py\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mbenchmark_manager\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrun_benchmark_workflow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m run_benchmark_workflow(\"TheBloke_vicuna-7B-1.1-GPTQ-4bit-128g\", 6666, group_size=128,\n\u001b[0m\u001b[1;32m      3\u001b[0m                            \u001b[0mprompt_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"long\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muser_tag\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"USER:\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                            assistant_tag=\"ASSISTANT:\", system_prefix=\"A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\", experiment_tag=\"vicuna\")\n",
      "\u001b[0;32m~/SageMaker/benchmark_manager.py\u001b[0m in \u001b[0;36mrun_benchmark_workflow\u001b[0;34m(model_name, portnum, group_size, maxnum, prompt_type, user_tag, assistant_tag, system_prefix, experiment_tag, working_directory)\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0;31m# Run the benchmark\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m     \u001b[0mrun_benchmark\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mexperiment_tag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxnum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mportnum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprompt_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muser_tag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0massistant_tag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msystem_prefix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0;31m# Once the benchmark has finished running, terminate the server process\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/SageMaker/benchmark_utils.py\u001b[0m in \u001b[0;36mrun_benchmark\u001b[0;34m(filename, maxnum, port, prompt_type, user_tag, assistant_tag, system_prefix, custom_completion)\u001b[0m\n\u001b[1;32m    106\u001b[0m             params = {\n\u001b[1;32m    107\u001b[0m                 \u001b[0;34m'task_id'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtask_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m                 'completion': custom_completion(\n\u001b[0m\u001b[1;32m    109\u001b[0m                     \u001b[0mproblems\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtask_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"prompt\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m                     \u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/SageMaker/benchmark_utils.py\u001b[0m in \u001b[0;36mgenerate_one_completion\u001b[0;34m(prompt_code, seed, port, prompt_type, user_tag, assistant_tag, system_prefix)\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0msuffix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'def'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mprompt_code\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"def\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"(\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"(\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m     \u001b[0mprompt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_prompt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt_code\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msuffix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprompt_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muser_tag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0massistant_tag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msystem_prefix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m     \u001b[0mcode_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mport\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mget_function_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcut_off_prefix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode_result\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"```python\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/SageMaker/benchmark_utils.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(prompt, seed, port)\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0mURI\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf'http://{HOST}:{port}/api/v1/generate'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;31m# Send the request and return the response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m     \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mURI\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mprompt\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'results'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m200\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/requests/api.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m    113\u001b[0m     \"\"\"\n\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    585\u001b[0m         }\n\u001b[1;32m    586\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 587\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    589\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    743\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    744\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 745\u001b[0;31m             \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    746\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    747\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/requests/models.py\u001b[0m in \u001b[0;36mcontent\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    897\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_content\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    898\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 899\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_content\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mb\"\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_content\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCONTENT_CHUNK_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34mb\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    900\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    901\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_content_consumed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/requests/models.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m()\u001b[0m\n\u001b[1;32m    814\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"stream\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    815\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 816\u001b[0;31m                     \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    817\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mProtocolError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    818\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mChunkedEncodingError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/urllib3/response.py\u001b[0m in \u001b[0;36mstream\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m    574\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    575\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_fp_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 576\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mamt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecode_content\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    577\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    578\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/urllib3/response.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m    517\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    518\u001b[0m                 \u001b[0mcache_content\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 519\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfp_closed\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34mb\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    520\u001b[0m                 if (\n\u001b[1;32m    521\u001b[0m                     \u001b[0mamt\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/http/client.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    461\u001b[0m             \u001b[0;31m# Amount is given, implement using readinto\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m             \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbytearray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 463\u001b[0;31m             \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadinto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    464\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtobytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    465\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/http/client.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    505\u001b[0m         \u001b[0;31m# connection, and the user is reading more bytes than will be provided\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m         \u001b[0;31m# (for example, reading in 1k chunks)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 507\u001b[0;31m         \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadinto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    508\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m             \u001b[0;31m# Ideally, we would raise IncompleteRead if the content-length\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    703\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 704\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    705\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    706\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from benchmark_manager import run_benchmark_workflow, run\n",
    "run_benchmark_workflow(\"TheBloke_vicuna-7B-1.1-GPTQ-4bit-128g\", 6666, group_size=128,\n",
    "                           prompt_type=\"long\", user_tag=\"USER:\", \n",
    "                           assistant_tag=\"ASSISTANT:\", system_prefix=\"A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\", experiment_tag=\"vicuna\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12b698d4-8b57-4f5f-bd3d-56c74b095e13",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_benchmark_workflow(\"TheBloke_wizardLM-7B-GPTQ\", 6666, group_size=128,\n",
    "                           prompt_type=\"long\", user_tag=\"USER:\", \n",
    "                           assistant_tag=\"ASSISTANT:\", system_prefix=\"\", experiment_tag=\"alpaca\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7ff04e5-8c9c-4600-ba61-bd1098773e4f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting server...\n",
      "INFO:\u001b[32mLoading TheBloke_wizardLM-7B-GPTQ...\u001b[0m\n",
      "WARNING:\u001b[33mMore than one .safetensors model has been found. The last one will be selected. It could be wrong.\u001b[0m\n",
      "INFO:\u001b[32mFound the following quantized model: models/TheBloke_wizardLM-7B-GPTQ/wizardLM-7B-GPTQ-4bit.latest.act-order.safetensors\u001b[0m\n",
      "INFO:\u001b[32mLoaded the model in 2.96 seconds.\n",
      "\u001b[0m\n",
      "Starting streaming server at ws://127.0.0.1:6667/api/v1/stream\n",
      "Starting API at http://127.0.0.1:6666/api\n",
      "Server started!\n",
      "Results will be written to: results/TheBloke_wizardLM-7B-GPTQ_custom.jsonl\n",
      "Processing Task 0 of 164\n",
      "def has_close_elements(numbers: List[float], threshold: float) -> bool:\n",
      "    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n",
      "        given threshold.\n",
      "        :param numbers: List[float]\n",
      "        :param threshold: float\n",
      "        :return: bool\n",
      "        \"\"\"\n",
      "    return len(set(numbers)) < 2 or abs(sum(numbers) - threshold) <= 1\n",
      "Processing Task 1 of 164\n",
      "def separate_paren_groups(paren_string: str) -> List[str]:\n",
      "    # Define a helper function to check if two brackets match\n",
      "    def are_brackets_matching(*args):\n",
      "        for i in range(len(args)):\n",
      "            if args[i]!= args[i+1]:\n",
      "                return False\n",
      "        return True\n",
      "    # Split the input string into lists of brackets based on their depth\n",
      "    depth = 0\n",
      "    result = []\n",
      "    for char in paren_string:\n",
      "        if char == '(':\n",
      "            depth += 1\n",
      "        elif char == ')':\n",
      "            depth -= 1\n",
      "        elif depth > 0:\n",
      "            result.append(char)\n",
      "        else:\n",
      "            result.extend([ch for ch in paren_string if ch not in'()'])\n",
      "    # Recursively split each sublist of brackets into individual strings\n",
      "    for i in range(len(result)-1):\n",
      "        if len(result[i]) > 1:\n",
      "            result[i], result[i+1] = result[i+1], result[i]\n",
      "    # Join the resulting lists of brackets into a single list of strings\n",
      "    return ['. '.join(lst) for lst in result]\n",
      "Processing Task 2 of 164\n",
      "def truncate_number(number: float) -> float:\n",
      "    \"\"\" Given a positive floating point number, it can be decomposed into\n",
      "    and integer part (largest integer smaller than given number) and decimals\n",
      "    (leftover part always smaller than 1).\n",
      "    Return the decimal part of the number.\n",
      "    >>> truncate_number(3.5)\n",
      "    0.5\n",
      "    \"\"\"\n",
      "    return number - int(round(number))\n",
      "Processing Task 3 of 164\n",
      "def below_zero(operations: T.List[int]) -> T.Union[bool, None]:\n",
      "    # Initialize a variable to keep track of whether the balance fell below zero or not\n",
      "    did_fall_below_zero = False\n",
      "    for operation in operations:\n",
      "        # Check if the current operation is a withdrawal operation\n",
      "        if operation < 0:\n",
      "            did_fall_below_zero = True\n",
      "            break\n",
      "    # Return True if the balance fell below zero, False otherwise\n",
      "    return did_fall_below_zero\n",
      "Processing Task 4 of 164\n",
      "def mean_absolute_deviation(numbers: typing.List[float]) -> float:\n",
      "    \"\"\" Calculate Mean Absolute Deviation for a given list of input numbers.\n",
      "        See documentation for details.\n",
      "    Returns:\n",
      "        The calculated value of Mean Absolute Deviation.\n",
      "    \"\"\"\n",
      "    # Initialize variables for calculating MAD\n",
      "    num_elements = len(numbers)\n",
      "    sum_of_squares = sum([num * num for num in numbers])\n",
      "    sum_of_absolutes = sum([abs(num) for num in numbers])\n",
      "    mean_value = sum_of_absolutes / num_elements\n",
      "    mad_value = math.sqrt(sum_of_squares - ((sum_of_absolutes ** 2) / num_elements))\n",
      "    return mad_value\n",
      "Processing Task 5 of 164\n",
      "def intersperse(numbers: List[int], delimeter: int) -> List[int]:\n",
      "    \"\"\"Insert a number 'delimeter' between every two consecutive elements of input list `numbers'\n",
      "    >>> intersperse([], 4)\n",
      "    []\n",
      "    >>> intersperse([1, 2, 3], 4)\n",
      "    [1, 4, 2, 4, 3]\n",
      "    \"\"\"\n",
      "Processing Task 6 of 164\n"
     ]
    }
   ],
   "source": [
    "# Make sure to pip install evalplus\n",
    "\n",
    "import os, signal\n",
    "from benchmark_utils import run_benchmark, run, extract_code\n",
    "from benchmark_manager import start_server\n",
    "\n",
    "model_name = \"TheBloke_wizardLM-7B-GPTQ\"\n",
    "portnum = 6666\n",
    "group_size=128\n",
    "\n",
    "server_process = start_server(model_name, portnum, group_size=group_size, \n",
    "                              working_directory='text-generation-webui') # Make sure server.py is in working_directory\n",
    "\n",
    "def my_completion(code, **kwargs):\n",
    "    prompt = \"Complete this code:\\n%s\\nASSISTANT:\" % code\n",
    "    results = extract_code(run(prompt, port=kwargs[\"port\"]))\n",
    "    print(results)\n",
    "    return results\n",
    "\n",
    "run_benchmark(model_name, port=portnum, custom_completion=my_completion, prompt_type = \"custom\")\n",
    "\n",
    "os.kill(server_process.pid, signal.SIGTERM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8bd1a5e-f349-46a4-b0c1-449c493536cb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "filename = \"results/TheBloke_wizardLM-7B-GPTQ_custom.jsonl\"\n",
    "\n",
    "result = subprocess.run([\"sudo\", \"/home/ec2-user/anaconda3/envs/pytorch_p39/bin/evalplus.evaluate\",\n",
    "                \"--dataset\", \"humaneval\", \"--samples\", filename, \"--i-just-wanna-run\"], \n",
    "                        text=True, capture_output=True, check=False)\n",
    "\n",
    "print(result.stdout, \"\\n\", result.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9dd23386-f2fc-4d4a-85eb-a89fd554aaa0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.kill(server_process.pid, signal.SIGTERM)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a35b5a37-cd8c-4390-a32c-539dc88cf900",
   "metadata": {},
   "source": [
    "You can also start a server, configure and load a model via the GUI, then benchmark against it (in a separate process or Jupyter notebook - see the advanced benchmark client notebook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7169fe61-9490-46ca-a4c0-b35a6bb3a024",
   "metadata": {},
   "outputs": [],
   "source": [
    "from benchmark_manager import start_server, block_log_server\n",
    "\n",
    "PORT = 6000 # remember this for the other notebook\n",
    "\n",
    "server_process = start_server(None, portnum=PORT, group_size=None, wbits=None,\n",
    "                              working_directory='text-generation-webui', public=True)\n",
    "block_log_server(server_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "985db790-f822-4557-8f99-c1032a3cb202",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p39",
   "language": "python",
   "name": "conda_pytorch_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}