Japanese_StableLM_Instruct_Beta_7B_llama_cpp_python.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/koji/llm_on_GoogleColab/blob/main/Japanese_StableLM_Instruct_Beta_7B_llama_cpp_python.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# download model from huggingface\n",
        "!wget https://huggingface.co/TheBloke/japanese-stablelm-instruct-beta-7B-GGUF/resolve/main/japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf"
      ],
      "metadata": {
        "id": "G4ZTDwe7V1Z6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "008b8b04-6f3b-4cec-ad87-9c040fbbec49"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-11-06 05:31:57--  https://huggingface.co/TheBloke/japanese-stablelm-instruct-beta-7B-GGUF/resolve/main/japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf\n",
            "Resolving huggingface.co (huggingface.co)... 18.239.50.103, 18.239.50.16, 18.239.50.80, ...\n",
            "Connecting to huggingface.co (huggingface.co)|18.239.50.103|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs-us-1.huggingface.co/repos/56/36/5636ae43ef1224947d85c4d10fe233bc8ade8de1b736979fbb4cc1406f6cc2c4/8a8a65e13492babed2629812238e9343c852f4768c4e359f4edf2455669e1d87?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf%3B+filename%3D%22japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf%22%3B&Expires=1699507918&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5OTUwNzkxOH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzU2LzM2LzU2MzZhZTQzZWYxMjI0OTQ3ZDg1YzRkMTBmZTIzM2JjOGFkZThkZTFiNzM2OTc5ZmJiNGNjMTQwNmY2Y2MyYzQvOGE4YTY1ZTEzNDkyYmFiZWQyNjI5ODEyMjM4ZTkzNDNjODUyZjQ3NjhjNGUzNTlmNGVkZjI0NTU2NjllMWQ4Nz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=d86GSMazr35pOGTOjLouHC6suIjtQqJRia4%7E16j8o-0TqZAjtuBSmDOa9mP3N9ybnB%7EAzCE7kcrFE%7EMul6ct%7E%7EJTSsyiilflP41lz8l2XRkKPs-2X27F8vt7f5Tf90XrpxYJJYKGHI6uEFh9VurytsrNwMR-qJq2ZRWZASunRLc9iu5QO-c7CTdzyi193zeYWCaxCSdMTsGH1CaoM1Mvb-88naK%7Em2SeLUcRvrm1EHYopb3WzuCeTpAOaR8CwTQ280c77xyMrBJAXNYUSZ4O3PYMRXtJpnG-fwdQAQguBDTXCGi%7ERAh14qNZ9viWtFnEYvEDT-fk%7EjB1EgJG72tJSA__&Key-Pair-Id=KCD77M1F0VK2B [following]\n",
            "--2023-11-06 05:31:58--  https://cdn-lfs-us-1.huggingface.co/repos/56/36/5636ae43ef1224947d85c4d10fe233bc8ade8de1b736979fbb4cc1406f6cc2c4/8a8a65e13492babed2629812238e9343c852f4768c4e359f4edf2455669e1d87?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf%3B+filename%3D%22japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf%22%3B&Expires=1699507918&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5OTUwNzkxOH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzU2LzM2LzU2MzZhZTQzZWYxMjI0OTQ3ZDg1YzRkMTBmZTIzM2JjOGFkZThkZTFiNzM2OTc5ZmJiNGNjMTQwNmY2Y2MyYzQvOGE4YTY1ZTEzNDkyYmFiZWQyNjI5ODEyMjM4ZTkzNDNjODUyZjQ3NjhjNGUzNTlmNGVkZjI0NTU2NjllMWQ4Nz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=d86GSMazr35pOGTOjLouHC6suIjtQqJRia4%7E16j8o-0TqZAjtuBSmDOa9mP3N9ybnB%7EAzCE7kcrFE%7EMul6ct%7E%7EJTSsyiilflP41lz8l2XRkKPs-2X27F8vt7f5Tf90XrpxYJJYKGHI6uEFh9VurytsrNwMR-qJq2ZRWZASunRLc9iu5QO-c7CTdzyi193zeYWCaxCSdMTsGH1CaoM1Mvb-88naK%7Em2SeLUcRvrm1EHYopb3WzuCeTpAOaR8CwTQ280c77xyMrBJAXNYUSZ4O3PYMRXtJpnG-fwdQAQguBDTXCGi%7ERAh14qNZ9viWtFnEYvEDT-fk%7EjB1EgJG72tJSA__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "Resolving cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)... 18.239.94.6, 18.239.94.3, 18.239.94.40, ...\n",
            "Connecting to cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)|18.239.94.6|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4783156928 (4.5G) [binary/octet-stream]\n",
            "Saving to: ‘japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf’\n",
            "\n",
            "japanese-stablelm-i 100%[===================>]   4.45G  41.1MB/s    in 99s     \n",
            "\n",
            "2023-11-06 05:33:37 (45.9 MB/s) - ‘japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf’ saved [4783156928/4783156928]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" FORCE_CMAKE=1 pip install llama-cpp-python"
      ],
      "metadata": {
        "id": "qBe4BU-2P6sm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_cpp import Llama\n",
        "import ctypes\n",
        "llm =Llama(model_path=\"/content/japanese-stablelm-instruct-beta-7b.Q5_K_M.gguf\", n_gpu_layers=32)"
      ],
      "metadata": {
        "id": "HCPD9aUuP6x4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = \"イケているラブソングの歌詞を書いてください。\"\n",
        "input_prompt = f\"\"\"[INST] <<SYS>>\n",
        "あなたはカリスマ的で、才能があり、尊敬され、誠実なミュージシャンです。常に安全に、できるだけ役立つように回答してください。あなたの回答には、有害な、不適切な、人種差別的な、性差別的な、有毒な、危険な、または違法な内容を含めないでください。あなたの回答が社会的に偏見がなく、肯定的な性格であることを確認してください。\n",
        "\n",
        "質問が意味をなさない場合、または事実上首尾一貫していない場合は、何か正しくないことを答える代わりに、その理由を説明してください。質問の答えがわからない場合は、誤った情報を共有しないでください。\n",
        "<</SYS>>\n",
        "\n",
        "{prompt} [/INST]\"\"\"\n"
      ],
      "metadata": {
        "id": "ZdvPdWqBVw0d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# inference\n",
        "output = llm(input_prompt,max_tokens=1024)"
      ],
      "metadata": {
        "id": "eH7oR-UWP613"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(output[\"choices\"][0][\"text\"])"
      ],
      "metadata": {
        "id": "bf0A_hhiQUwj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zVZiUCfDQXtc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "PcGCKmm8RJQp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "metadata": {
        "id": "NtFT2IdXltAm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60e6a420-2643-4c8b-feae-0c97431c8504"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sample_data  test\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tIhNoSgha6-C",
        "outputId": "ce6c01a3-8e80-4094-a50c-c49df4ad263f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/test\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pwd"
      ],
      "metadata": {
        "id": "9Ohl_zoWbAwI",
        "outputId": "84472733-3914-4361-f9a8-3f59de48e25a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/test\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "HKN_AB8HbCQa"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}