Corrective_RAG_warmup.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMIvgPqLOYe6nurHlxvgGFz",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shakilkhan24/Playground_DL/blob/main/Corrective_RAG_warmup.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2eAFVFnpG9LA"
      },
      "outputs": [],
      "source": [
        "api_key=' '"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain\n",
        "!pip install langchain-openai"
      ],
      "metadata": {
        "id": "rZvQyp-wHDCg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install unstructured"
      ],
      "metadata": {
        "id": "cgYNW7nWHFc8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb"
      ],
      "metadata": {
        "id": "iA3MZtrhHYCD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# web page scrapping and storing into vector database\n",
        "\n",
        "\n",
        "```\n",
        "# splitting , embedding\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "fswQfsATX4gW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from bs4 import BeautifulSoup\n",
        "import os\n",
        "import requests\n",
        "import chromadb\n",
        "from typing import List, Dict\n",
        "from langchain_community.document_loaders import UnstructuredHTMLLoader\n",
        "from langchain_community.vectorstores import Chroma\n",
        "from langchain_community.embeddings import OpenAIEmbeddings\n",
        "from langchain_community.chat_models import ChatOpenAI\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_openai import OpenAI\n",
        "from langchain import PromptTemplate\n",
        "\n",
        "url='https://huggingface.co/learn/nlp-course/chapter3/4'\n",
        "html = requests.get(url).text\n",
        "with open('explore.html', 'w') as f:\n",
        "    f.write(html)\n",
        "\n",
        "chat = ChatOpenAI(temperature=0, openai_api_key=api_key)\n",
        "embedding = OpenAIEmbeddings(openai_api_key=api_key)\n",
        "\n",
        "loader = UnstructuredHTMLLoader(\"explore.html\")\n",
        "docs = loader.load()\n",
        "\n",
        "splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=1000,\n",
        "    chunk_overlap=200,\n",
        " )\n",
        "\n",
        "text_chunks = splitter.split_documents(docs)\n",
        "\n",
        "db = Chroma.from_documents(text_chunks, embedding)\n",
        "chat_model = ChatOpenAI(openai_api_key=api_key)"
      ],
      "metadata": {
        "id": "dnO4c7E5HSvU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# we can specify multiple chuncks\n",
        "\n",
        "\n",
        "```\n",
        "# usually we take the first one\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "awKlHlSBYPh_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(text_chunks[3].page_content)"
      ],
      "metadata": {
        "id": "jBYIOnCQIDM8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# first checking **retrieved** data is query relevant or not.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "# directing llm to return YES or NO as string\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "Ba6TrJyvYZTz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompttemplate1 = \"\"\"\n",
        "Is this retrieved data : [{retrieved_data1}], is relevant accoroding to the query: [{query1}]?\n",
        "Return a simple boolean variable 'yes' or 'no'.\n",
        "\"\"\"\n",
        "# returning a boolean variable yes or no .\n",
        "\n",
        "x='tell me about the soil of mars planet?'\n",
        "retrieved_data1=db.similarity_search(x)[0]\n",
        "\n",
        "prompt1=PromptTemplate.from_template(prompttemplate1)\n",
        "prompt1=prompt1.format(retrieved_data1=retrieved_data1,query1=x)\n"
      ],
      "metadata": {
        "id": "W8p_qJajIE3G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "yes_or_no=chat_model.predict(prompt1)\n"
      ],
      "metadata": {
        "id": "lS1V8R9sPYln"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "yes_or_no"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "qFUBg5YEQ_9M",
        "outputId": "2ae1e7cd-0534-4225-accd-49583742e4d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'No'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(yes_or_no)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gKRA4aFIPor8",
        "outputId": "56f761e4-7b85-40a1-ff7f-b4ed2368516c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "str"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(retrieved_data1.page_content)"
      ],
      "metadata": {
        "id": "7Z-8jvIVLXsf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U langchain-community tavily-python"
      ],
      "metadata": {
        "id": "DGFgbdQZSDn0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# TAVILY_API_KEY='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'\n",
        "import getpass\n",
        "import os\n",
        "\n",
        "os.environ[\"TAVILY_API_KEY\"] = getpass.getpass()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HW33cyWqScWe",
        "outputId": "450ecf88-e400-470d-f7bb-38d2e4da4000"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Tavily** is for web search"
      ],
      "metadata": {
        "id": "5yNQEB4RYxeU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_community.tools.tavily_search import TavilySearchResults\n",
        "\n",
        "tool = TavilySearchResults()"
      ],
      "metadata": {
        "id": "XXIdyKZRSCOU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt_template2=\"\"\"\n",
        "answer the question based on the data: [{retrieved_data}, accoroding to the prompt: [{prompt}]]\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "g75F-UCMVG5C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# If **YES** , then predicting based on **Retrieved_data**\n",
        "\n",
        "# If **NO**, then predicting based on **web_search** data\n",
        "\n",
        "\n",
        "```\n",
        "# **x** is the main question we started with\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "j2tBY601Y4a4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# if yes , then predicting based on the retrieved data , additionally model's pretrained knowledge\n",
        "# if no, then go for web searching\n",
        "\n",
        "\n",
        "if yes_or_no=='yes':\n",
        "  result=chat_model.invoke(prompt_template2.format(retrieved_data=retrieved_data1,prompt=x))\n",
        "else:\n",
        "  search_result=tool.invoke({'query':x})\n",
        "  print(search_result)\n",
        "  result=chat_model.invoke(prompt_template2.format(retrieved_data=search_result,prompt=x))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QyXDP2CKV64Z",
        "outputId": "98d70170-f0ed-464f-83de-e0400d6dc65e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'url': 'https://scitechdaily.com/geologists-simulate-martian-soil-conditions-to-figure-out-how-to-grow-plants-on-mars/', 'content': 'A difficult task Using simulated Martian soils, Fackrell and fellow researchers have found the textures of artificial simulants to be crusty and dried which may reflect some unexpected conditions of Mars soils that make them more difficult to use. These challenges add up to a very difficult, though not impossible task.'}, {'url': 'https://geoscience.blog/exploring-the-science-of-martian-soil-unveiling-the-secrets-to-replicating-the-red-planets-unique-composition-on-earth/', 'content': '1. Select basaltic material The first step is to obtain basaltic material that closely resembles the composition of the Martian regolith. Basalt rocks similar to those found on Mars can be sourced from volcanic regions on Earth.'}, {'url': 'https://academic.oup.com/astrogeo/article/57/2/2.18/2468646', 'content': 'Watney has water and it is safe to assume that any humans on Mars would have a reliable water supply. After water, carbon is the most limiting nutrient. On Earth, plants can survive in carbon dioxide concentrations in the soil ranging from 400 ppm (ambient) to 10 000 ppm (extreme), and an atmospheric carbon dioxide that is currently around 400 ...'}, {'url': 'https://en.wikipedia.org/wiki/Martian_soil', 'content': 'Martian soil is the fine regolith (a blanket of unconsolidated, loose, heterogeneous superficial deposits covering solid rock) found on the surface of Mars. Its properties can differ significantly from those of terrestrial soil, including its toxicity due to the presence of perchlorates.'}, {'url': 'https://cen.acs.org/articles/96/i1/build-settlements-Mars-ll-need.html', 'content': \"Mars regolith is mostly silicon dioxide and ferric oxide, with a fair amount of aluminum oxide, calcium oxide, and sulfur oxide. The composition varies from place to place on the planet's surface because of variability in asteroid collisions and the weathering by wind and water, in ancient oceans and in some modern water flows.\"}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# we need to ***parsing*** our results"
      ],
      "metadata": {
        "id": "nJO8S9fVZaac"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "search_result"
      ],
      "metadata": {
        "id": "j-ywo3IwXZOC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ODo0Sqv4W5mL",
        "outputId": "350d3bf8-c95c-4a00-fe92-7ca6ad3cef79"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "AIMessage(content='Sorry, I cannot provide information about the soil of Mars planet based on the given data about fine-tuning models and training loops.')"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "openai_api_key=''"
      ],
      "metadata": {
        "id": "lltuvSvAbpCI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Entire code\n",
        "# make ensure openai_api and tavily_api is set\n",
        "\n",
        "\n",
        "```\n",
        "# just give any webpage and question , it would generate solution for you\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "8ltYAou-eJ4W"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from typing import List, Dict\n",
        "from langchain_community.document_loaders import UnstructuredHTMLLoader\n",
        "from langchain_community.vectorstores import Chroma\n",
        "from langchain_community.embeddings import OpenAIEmbeddings\n",
        "from langchain_community.chat_models import ChatOpenAI\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_openai import OpenAI\n",
        "from langchain import PromptTemplate\n",
        "\n",
        "def process_data(url: str, x: str) -> str:\n",
        "    html = requests.get(url).text\n",
        "    with open('explore.html', 'w') as f:\n",
        "        f.write(html)\n",
        "\n",
        "    chat = ChatOpenAI(temperature=0, openai_api_key=openai_api_key)\n",
        "    embedding = OpenAIEmbeddings(openai_api_key=openai_api_key)\n",
        "\n",
        "    loader = UnstructuredHTMLLoader(\"explore.html\")\n",
        "    docs = loader.load()\n",
        "\n",
        "    splitter = RecursiveCharacterTextSplitter(\n",
        "        chunk_size=1000,\n",
        "        chunk_overlap=200,\n",
        "    )\n",
        "\n",
        "    text_chunks = splitter.split_documents(docs)\n",
        "\n",
        "    db = Chroma.from_documents(text_chunks, embedding)\n",
        "    chat_model = ChatOpenAI(openai_api_key=openai_api_key)\n",
        "\n",
        "    prompttemplate1 = \"\"\"\n",
        "    Is this retrieved data: [{retrieved_data1}], relevant according to the query: [{query1}]?\n",
        "    Return a simple boolean variable 'yes' or 'no'.\n",
        "    \"\"\"\n",
        "\n",
        "    # returning a boolean variable yes or no.\n",
        "    retrieved_data1 = db.similarity_search(x)[0]\n",
        "    prompt1 = PromptTemplate.from_template(prompttemplate1)\n",
        "    prompt1 = prompt1.format(retrieved_data1=retrieved_data1, query1=x)\n",
        "\n",
        "    from langchain_community.tools.tavily_search import TavilySearchResults\n",
        "\n",
        "    tool = TavilySearchResults()\n",
        "    prompt_template2 = \"\"\"\n",
        "    Answer the question based on the data: [{retrieved_data}], according to the prompt: [{prompt}],\n",
        "    also you can add more information from your knowledge (pretrained) , as if answer tends to be perfect or accurate or organized etc\n",
        "    \"\"\"\n",
        "\n",
        "    # If yes, then predicting based on the retrieved data, additionally model's pretrained knowledge\n",
        "    # If no, then go for web searching\n",
        "   # Placeholder value, replace with actual logic\n",
        "\n",
        "    if yes_or_no.lower() == 'yes':\n",
        "        result = chat_model.invoke(prompt_template2.format(retrieved_data=retrieved_data1, prompt=x))\n",
        "    else:\n",
        "        search_result = tool.invoke({'query': x})\n",
        "        print(search_result)\n",
        "        result = chat_model.invoke(prompt_template2.format(retrieved_data=search_result, prompt=x))\n",
        "\n",
        "    return result"
      ],
      "metadata": {
        "id": "IR1QTtf0bKXD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "url='https://huggingface.co/learn/nlp-course/chapter3/4'\n",
        "question='what is the elements of mars soil?'\n",
        "print(process_data(url,question))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rqu45NUDdGZk",
        "outputId": "0ebff086-b899-4def-e297-e41a81d59e13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'url': 'https://www.spacechatter.com/2024/01/29/solvents-building-materials-mars/', 'content': 'January 29, 2024 by Space Chatter Wire Will humans be able to build on the Moon and Mars? Thanks to Washington State University researchers, the sci-fi fantasy is moving closer to reality. Scientists discovered powerful liquid solvents capable of extracting vital materials from lunar and Martian rock dust for 3D printing.'}, {'url': 'https://www.newscientist.com/article/dn22372-black-glass-holds-first-mars-soil-sample-on-earth/', 'content': 'The team says weakly acidic water may have leached rare-earth elements from Martian soil and deposited them in fractures in surface rocks. Heat from the asteroid impact that ejected Tissint melted ...'}, {'url': 'https://www.nature.com/articles/nature03637', 'content': 'Author information\\nAuthors and Affiliations\\nJet Propulsion Laboratory, California Institute of Technology, California, 91109, Pasadena, USA\\nAlbert S. Yen,Â\\xa0Joy A. CrispÂ\\xa0&Â\\xa0Diana Blaney\\nMax Planck Institut fÃ¼r Chemie,\\nRalf Gellert,Â\\xa0Johannes BrÃ¼ckner,Â\\xa0Rudi RiederÂ\\xa0&Â\\xa0Jutta Zipfel\\nJohannes Gutenberg University, D-55128, Mainz, Germany\\nChristian SchrÃ¶der,Â\\xa0GÃ¶star KlingelhÃ¶ferÂ\\xa0&Â\\xa0Daniel Rodionov\\nNASA Johnson Space Center, Houston, Texas, 77058, USA\\nRichard V. MorrisÂ\\xa0&Â\\xa0Douglas W. Ming\\nCornell University, Department of Astronomy, New York, 14853, Ithaca, USA\\nJames F. Bell IIIÂ\\xa0&Â\\xa0Steven W. Squyres\\nDepartment of Geological Sciences, Arizona State University, Tempe, Arizona, 85287, USA\\nAmy T. Knudson,Â\\xa0Philip R. ChristensenÂ\\xa0&Â\\xa0Michael Wyatt\\nLockheed Martin Corporation, Littleton, Colorado, 80127, USA\\nBenton C. Clark\\nWashington University, Missouri, 63130, Saint Louis, USA\\nRaymond E. Arvidson,Â\\xa0Larry A. Haskin,Â\\xa0Bradley L. JoliffÂ\\xa0&Â\\xa0Alian Wang\\nNASA Ames Research Center, Moffett Field, California, 94035, USA\\nDavid J. DesMarais\\nCompanhia Vale do Rio Doce, 29030-900, Rio de Janeiro, Brazil\\nPaulo A. de Souza Jr\\nEnrico Fermi Institute, University of Chicago, Illinois, 60637, Chicago, USA\\nThanasis E. Economou\\nDepartment of Earth and Planetary Sciences, University of Tennessee, Tennessee, 37996, Knoxville, USA\\nAmitabha GhoshÂ\\xa0&Â\\xa0Harry Y. McSween\\nState University of New York, Department of Geosciences, New York, 11794, Stony Brook, USA\\nBrian C. Hahn,Â\\xa0Joel A. Hurowitz,Â\\xa0Scott M. McLennanÂ\\xa0&Â\\xa0Nicholas J. Tosca\\nUS Geological Survey, Arizona, 86001, Flagstaff, USA\\nKenneth E. Herkenhoff,Â\\xa0 Advertisement\\nAn integrated view of the chemistry and mineralogy of martian soils\\nNature\\nvolumeÂ\\xa0436,Â\\xa0pages 49â€“54 (2005)Cite this article\\n3262 Accesses\\n299 Citations\\n3 Altmetric\\nMetrics details\\nAn Erratum to this article was published on 11 August 2005\\n ISSN 0028-0836 (print)\\nnature.com sitemap\\nAbout Nature Portfolio\\nDiscover content\\nPublishing policies\\nAuthor & Researcher services\\nLibraries & institutions\\nAdvertising & partnerships\\nProfessional development\\nRegional websites\\n You can also search for this author in\\nPubMedÂ\\xa0Google Scholar\\nYou can also search for this author in\\nPubMedÂ\\xa0Google Scholar\\nCorresponding author\\nCorrespondence to\\nAlbert S. Yen.\\n https://doi.org/10.1038/nature03637\\nDownload citation\\nReceived: 20 November 2004\\nAccepted: 08 April 2005\\nIssue Date: 07 July 2005\\nDOI: https://doi.org/10.1038/nature03637\\nShare this article\\nAnyone you share the following link with will be able to read this content:\\n'}, {'url': 'https://phys.org/news/2024-01-scientists-biocatalytic-reactor-detoxifying-mars.html', 'content': 'With concentrations of about 0.5% found in these northern plain soils, scientists realized why previous attempts to find organic molecules in Martian soil had failed.'}, {'url': 'https://chicagolandgardening.com/gardening-basics/understanding-soil/what-is-mars-soil-made-of/', 'content': 'Mars, often referred to as the \"Red Planet,\" possesses a unique and complex soil composition that differs significantly from Earth\\'s soil. This distinct composition is a result of various factors including the planet\\'s geological processes, atmospheric conditions, and interactions with cosmic radiation.'}]\n",
            "content=\"Based on the data provided, the elements present in Mars soil include rare-earth elements that may have been leached by weakly acidic water and deposited in fractures in surface rocks. Additionally, scientists have found organic molecules in Martian soil at concentrations of about 0.5% in northern plain soils. \\n\\nFurthermore, Mars soil is known to have a unique and complex composition that differs significantly from Earth's soil. This composition is a result of various factors such as the planet's geological processes, atmospheric conditions, and interactions with cosmic radiation. \\n\\nIn summary, the elements found in Mars soil include rare-earth elements and organic molecules, and the composition of Mars soil is influenced by geological processes, atmospheric conditions, and cosmic radiation.\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-138lUKzar1C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# entire pipeline here ,,, work on it ___"
      ],
      "metadata": {
        "id": "9Ih4fV1wNU41"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from urllib.parse import urlparse, urljoin\n",
        "from langchain_community.tools.tavily_search import TavilySearchResults\n",
        "from langchain_community.loader import UnstructuredHTMLLoader\n",
        "from langchain_community.splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.chroma import Chroma\n",
        "from langchain_community.prompt_template import PromptTemplate\n",
        "from langchain_community.chat_openai import ChatOpenAI\n",
        "from langchain_community.openai_embeddings import OpenAIEmbeddings\n",
        "import logging\n",
        "import os\n",
        "\n",
        "class WebsiteProcessor:\n",
        "    def __init__(self, openai_api_key):\n",
        "        self.openai_api_key = openai_api_key\n",
        "        self.chat_model = ChatOpenAI(temperature=0, openai_api_key=self.openai_api_key)\n",
        "        self.embedding = OpenAIEmbeddings(openai_api_key=self.openai_api_key)\n",
        "        self.logger = self.setup_logger('website_processor')\n",
        "\n",
        "    def setup_logger(self, logger_name, log_file='website_processor.log'):\n",
        "        logger = logging.getLogger(logger_name)\n",
        "        logger.setLevel(logging.DEBUG)\n",
        "\n",
        "        file_handler = logging.FileHandler(log_file)\n",
        "        formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(message)s')\n",
        "        file_handler.setFormatter(formatter)\n",
        "\n",
        "        logger.addHandler(file_handler)\n",
        "        return logger\n",
        "\n",
        "    def bfs_crawl(self, url, limit):\n",
        "        try:\n",
        "            visited = set()\n",
        "            queue = [url]\n",
        "            count = 0\n",
        "            extracted_links = []\n",
        "\n",
        "            while queue and count < limit:\n",
        "                current_url = queue.pop(0)\n",
        "                visited.add(current_url)\n",
        "\n",
        "                try:\n",
        "                    response = requests.get(current_url)\n",
        "                    response.raise_for_status()  # Raise an exception for non-2xx status codes\n",
        "                except requests.exceptions.RequestException as e:\n",
        "                    self.logger.warning(f\"Error while crawling {current_url}: {e}\")\n",
        "                    continue\n",
        "\n",
        "                soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "                for link in soup.find_all('a'):\n",
        "                    href = link.get('href')\n",
        "                    if href:\n",
        "                        absolute_url = urljoin(current_url, href)\n",
        "                        parsed_url = urlparse(absolute_url)\n",
        "\n",
        "                        if parsed_url.netloc == urlparse(url).netloc:\n",
        "                            if absolute_url not in visited and absolute_url not in queue:\n",
        "                                queue.append(absolute_url)\n",
        "                                visited.add(absolute_url)\n",
        "                                count += 1\n",
        "                                extracted_links.append(absolute_url)\n",
        "\n",
        "                self.logger.info(f\"Crawled: {current_url}\")\n",
        "\n",
        "            return extracted_links\n",
        "        except Exception as e:\n",
        "            self.logger.error(f\"Error while crawling: {e}\")\n",
        "            raise\n",
        "\n",
        "    def str_func(self, s: str):\n",
        "        # Implement your data cleaning/preprocessing logic here\n",
        "        return s\n",
        "\n",
        "    def extract_info_from_links(self, links):\n",
        "        try:\n",
        "            docs = ''\n",
        "            for link in links:\n",
        "                try:\n",
        "                    loader = UnstructuredHTMLLoader(link)\n",
        "                    d = loader.load()\n",
        "                    docs += self.str_func(d[0].page_content)\n",
        "                except Exception as e:\n",
        "                    self.logger.warning(f\"Error while processing {link}: {e}\")\n",
        "                    continue\n",
        "            return docs\n",
        "        except Exception as e:\n",
        "            self.logger.error(f\"Error while extracting information from links: {e}\")\n",
        "            raise\n",
        "\n",
        "    def html_converter(self, docs: str):\n",
        "        try:\n",
        "            with open('entire_text.txt', 'w', encoding='utf-8') as f:\n",
        "                f.write(docs)\n",
        "        except Exception as e:\n",
        "            self.logger.error(f\"Error while converting to HTML: {e}\")\n",
        "            raise\n",
        "\n",
        "    def process_html_file(self, html_file_path, query):\n",
        "        try:\n",
        "            with open(html_file_path, 'r', encoding='utf-8') as f:\n",
        "                html = f.read()\n",
        "\n",
        "            loader = UnstructuredHTMLLoader(html_file_path)\n",
        "            docs = loader.load()\n",
        "\n",
        "            splitter = RecursiveCharacterTextSplitter(\n",
        "                chunk_size=1000,\n",
        "                chunk_overlap=200,\n",
        "            )\n",
        "\n",
        "            text_chunks = splitter.split_documents(docs)\n",
        "\n",
        "            db = Chroma.from_documents(text_chunks, self.embedding)\n",
        "\n",
        "            prompttemplate1 = \"\"\"\n",
        "            Is this retrieved data: [{retrieved_data1}], relevant according to the query: [{query1}]?\n",
        "            Return a simple boolean variable 'yes' or 'no'.\n",
        "            \"\"\"\n",
        "\n",
        "            retrieved_data1 = db.similarity_search(query)[0]\n",
        "            prompt1 = PromptTemplate.from_template(prompttemplate1)\n",
        "            prompt1 = prompt1.format(retrieved_data1=retrieved_data1, query1=query)\n",
        "            yes_or_no = self.chat_model.invoke(prompt1)\n",
        "\n",
        "            tool = TavilySearchResults()\n",
        "            prompt_template2 = \"\"\"\n",
        "            Answer the question based on the data: [{retrieved_data}], according to the prompt: [{prompt}],\n",
        "            also you can add more information from your knowledge (pretrained) , as if answer tends to be perfect or accurate or organized etc\n",
        "            \"\"\"\n",
        "\n",
        "            if yes_or_no.lower() == 'yes':\n",
        "                result = self.chat_model.invoke(prompt_template2.format(retrieved_data=retrieved_data1, prompt=query))\n",
        "            else:\n",
        "                search_result = tool.invoke({'query': query})\n",
        "                print(search_result)\n",
        "                result = self.chat_model.invoke(prompt_template2.format(retrieved_data=search_result, prompt=query))\n",
        "\n",
        "            return result\n",
        "        except Exception as e:\n",
        "            self.logger.error(f\"Error while processing HTML file: {e}\")\n",
        "            raise\n",
        "\n",
        "    def extract_code_with_language(self, string):\n",
        "        try:\n",
        "            pattern = r\"\\`\\`\\`(\\[\\\\w\\\\-#\\]+)?\\\\n(\\[\\\\s\\\\S\\]\\*?)\\\\n?\\`\\`\\`\"\n",
        "            matches = re.findall(pattern, string)\n",
        "            code_with_language = []\n",
        "            for match in matches:\n",
        "                language_name = match[0].strip() if match[0] else None\n",
        "                code = match[1]\n",
        "                code_with_language.append({'language': language_name, 'code': code})\n",
        "            return code_with_language\n",
        "        except Exception as e:\n",
        "            self.logger.error(f\"Error while extracting code with language: {e}\")\n",
        "            raise\n",
        "\n",
        "# Example usage:\n",
        "if __name__ == \"__main__\":\n",
        "    openai_api_key = os.environ.get('OPENAI_API_KEY', 'your_openai_api_key')\n",
        "    processor = WebsiteProcessor(openai_api_key)\n",
        "    website_url = \"https://example.com\"\n",
        "    num_links = 100\n",
        "    links = processor.bfs_crawl(website_url, num_links)\n",
        "    website_content = processor.extract_info_from_links(links)\n",
        "    processor.html_converter(website_content)\n",
        "    query = \"What is the capital of France?\"\n",
        "    result = processor.process_html_file('entire_text.txt', query)\n",
        "    print(result)"
      ],
      "metadata": {
        "id": "HUz5eQwqar8G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# for code snippet generation , extracting the code parts\n",
        "# output parsing technicque little bit"
      ],
      "metadata": {
        "id": "mVznnmP6dfV2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def extract_code_with_language(string):\n",
        "    pattern = r\"```([\\w\\-#]+)?\\n([\\s\\S]*?)\\n?```\"\n",
        "    matches = re.findall(pattern, string)\n",
        "    code_with_language = []\n",
        "\n",
        "    for match in matches:\n",
        "        language_name = match[0].strip() if match[0] else None\n",
        "        code = match[1]\n",
        "        code_with_language.append({'language': language_name, 'code': code})\n",
        "\n",
        "    return code_with_language\n",
        "\n",
        "# use fronted techniques to show the code snippets in a gorgeous way..."
      ],
      "metadata": {
        "id": "J5CSMC5Za5-G"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}