diff --git a/.env.template b/.env.template index b854e82..25101c9 100644 --- a/.env.template +++ b/.env.template @@ -7,6 +7,7 @@ OPENAI_API_BASE=https://YOUR_AZURE_OPENAI_RESOURCE.openai.azure.com/ OPENAI_API_KEY=YOUR_AZURE_OPENAI_API_KEY OPENAI_TEMPERATURE=0.7 OPENAI_MAX_TOKENS=-1 +AZURE_CLOUD=AzureCloud # AzureCloud or AzureChinaCloud VECTOR_STORE_TYPE=AzureSearch AZURE_SEARCH_SERVICE_NAME=YOUR_AZURE_SEARCH_SERVICE_NAME AZURE_SEARCH_ADMIN_KEY=YOUR_AZURE_SEARCH_ADMIN_KEY diff --git a/.gitignore b/.gitignore index 06b74d1..c6be6dd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,8 @@ code/embeddings_text.csv code/utilities/__pycache__ .env -__pycache__ \ No newline at end of file +__pycache__ +.vscode +WebApp.Dockerfile +BatchProcess.Dockerfile +.gitignore \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..67aaf07 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "azureFunctions.deploySubpath": "code", + "azureFunctions.scmDoBuildDuringDeployment": true, + "azureFunctions.pythonVenv": ".venv", + "azureFunctions.projectLanguage": "Python", + "azureFunctions.projectRuntime": "~4", + "debug.internalConsoleOptions": "neverOpen" +} \ No newline at end of file diff --git a/README.md b/README.md index 30f1069..472a274 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ If you want to use a Chat based deployment (gpt-35-turbo or gpt-4-32k or gpt-4), You have multiple options to run the code: - [Deploy on Azure (WebApp + Batch Processing) with Azure Cognitive Search](#deploy-on-azure-webapp--batch-processing-with-azure-cognitive-search) - [Deploy on Azure (WebApp + Azure Cache for Redis + Batch Processing)](#deploy-on-azure-webapp--azure-cache-for-redis-enterprise--batch-processing) -- [Deploy on Azure (WebApp + Redis Stack + Batch Processing)](#deploy-on-azure-webapp--redis-stack--batch-processing) +- [Deploy on Azure/Azure China (WebApp + Redis Stack + Batch Processing)](#deploy-on-azureazure-china-webapp--redis-stack--batch-processing) +- [Deploy on Azure/Azure China (WebApp + Azure PostgreSQL + Batch Processing)](#deploy-on-azureazure-china-webapp--azure-postgresql--batch-processing) - [Run everything locally in Docker (WebApp + Redis Stack + Batch Processing)](#run-everything-locally-in-docker-webapp--redis-stack--batch-processing) - [Run everything locally in Python with Conda (WebApp only)](#run-everything-locally-in-python-with-conda-webapp-only) - [Run everything locally in Python with venv](#run-everything-locally-in-python-with-venv) @@ -71,8 +72,9 @@ Please be aware that you still need: You will add the endpoint and access key information for these resources when deploying the template. -## Deploy on Azure (WebApp + Redis Stack + Batch Processing) -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fruoccofabrizio%2Fazure-open-ai-embeddings-qna%2Fmain%2Finfrastructure%2Fdeployment.json) +## Deploy on Azure/Azure China (WebApp + Redis Stack + Batch Processing) +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fruoccofabrizio%2Fazure-open-ai-embeddings-qna%2Fmain%2Finfrastructure%2Fdeployment.json) +[![Deploy to Azure](https://aka.ms/deploytoazurechinabutton)](https://portal.azure.cn/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fcyberflying%2Fazure-open-ai-embeddings-qna%2Fmain%2Finfrastructure%2Fdeployment_azcn.json) Click on the Deploy to Azure button and configure your settings in the Azure Portal as described in the [Environment variables](#environment-variables) section. @@ -83,6 +85,15 @@ Please be aware that you need: - an existing Form Recognizer Resource (OPTIONAL - if you want to extract text out of documents) - an existing Translator Resource (OPTIONAL - if you want to translate documents) +## Deploy on Azure/Azure China (WebApp + Azure PostgreSQL + Batch Processing) +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fcyberflying%2Fazure-open-ai-embeddings-qna%2Fmain%2Finfrastructure%2Fdeployment_pg.json) +[![Deploy to Azure](https://aka.ms/deploytoazurechinabutton)](https://portal.azure.cn/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fcyberflying%2Fazure-open-ai-embeddings-qna%2Fmain%2Finfrastructure%2Fdeployment_pg_azcn.json) + +Click on the Deploy to Azure button and configure your settings in the Azure Portal as described in the [Environment variables](#environment-variables) section. + +![Architecture](docs/architecture_pg.png) + + ## Run everything locally in Docker (WebApp + Redis Stack + Batch Processing) First, clone the repo: @@ -268,11 +279,17 @@ Here is the explanation of the parameters: |OPENAI_EMBEDDINGS_ENGINE_QUERY | text-embedding-ada-002 | Embedding engine for query deployed in your Azure OpenAI resource| |OPENAI_API_BASE | https://YOUR_AZURE_OPENAI_RESOURCE.openai.azure.com/ | Your Azure OpenAI Resource name. Get it in the [Azure Portal](https://portal.azure.com)| |OPENAI_API_KEY| YOUR_AZURE_OPENAI_KEY | Your Azure OpenAI API Key. Get it in the [Azure Portal](https://portal.azure.com)| -|OPENAI_TEMPERATURE|0.7| Azure OpenAI Temperature | +|OPENAI_TEMPERATURE|0.1| Azure OpenAI Temperature | |OPENAI_MAX_TOKENS|-1| Azure OpenAI Max Tokens | -|VECTOR_STORE_TYPE| AzureSearch | Vector Store Type. Use AzureSearch for Azure Cognitive Search, leave it blank for Redis or Azure Cache for Redis Enterprise| +|AZURE_CLOUD|AzureCloud| Azure Cloud to use. AzureCloud for Azure Global, AzureChinaCloud for Azure China | +|VECTOR_STORE_TYPE| PGVector | Vector Store Type. Use AzureSearch for Azure Cognitive Search, PGVector for Azure PostgreSQL, leave it blank for Redis or Azure Cache for Redis Enterprise| |AZURE_SEARCH_SERVICE_NAME| YOUR_AZURE_SEARCH_SERVICE_URL | Your Azure Cognitive Search service name. Get it in the [Azure Portal](https://portal.azure.com)| |AZURE_SEARCH_ADMIN_KEY| AZURE_SEARCH_ADMIN_KEY | Your Azure Cognitive Search Admin key. Get it in the [Azure Portal](https://portal.azure.com)| +|PGVECTOR_HOST|Your_PG_NAME.postgres.database.azure.com or Your_PG_NAME.postgres.database.chinacloudapi.cn +|PGVECTOR_PORT|5432 +|PGVECTOR_DATABASE|YOUR_PG_DATABASE +|PGVECTOR_USER|YOUR_PG_USER +|PGVECTOR_PASSWORD|YOUR_PG_PASSWORD |REDIS_ADDRESS| api | URL for Redis Stack: "api" for docker compose| |REDIS_PORT | 6379 | Port for Redis | |REDIS_PASSWORD| redis-stack-password | OPTIONAL - Password for your Redis Stack| diff --git a/code/OpenAI_Queries.py b/code/OpenAI_Queries.py index 9965413..b099111 100644 --- a/code/OpenAI_Queries.py +++ b/code/OpenAI_Queries.py @@ -47,19 +47,10 @@ def check_deployment(): Then restart your application. """) st.error(traceback.format_exc()) - #\ 4. Check if the Redis is working with previous version of data + #\ 4. Check if the VectorStore is working with previous version of data try: llm_helper = LLMHelper() - if llm_helper.vector_store_type != "AzureSearch": - if llm_helper.vector_store.check_existing_index("embeddings-index"): - st.warning("""Seems like you're using a Redis with an old data structure. - If you want to use the new data structure, you can start using the app and go to "Add Document" -> "Add documents in Batch" and click on "Convert all files and add embeddings" to reprocess your documents. - To remove this working, please delete the index "embeddings-index" from your Redis. - If you prefer to use the old data structure, please change your Web App container image to point to the docker image: fruocco/oai-embeddings:2023-03-27_25. - """) - else: - st.success("Redis is working!") - else: + if llm_helper.vector_store_type == "AzureSearch": try: llm_helper.vector_store.index_exists() st.success("Azure Cognitive Search is working!") @@ -69,6 +60,26 @@ def check_deployment(): Then restart your application. """) st.error(traceback.format_exc()) + elif llm_helper.vector_store_type == "PGVector": + try: + llm_helper.vector_store.__post_init__() + st.success("PGVector is working!") + except Exception as e: + st.error("""PGVector is not working. + Please check your Azure PostgreSQL server, database, user name and password in the App Settings. + Make sure the network settings(firewall rule) allow your app to access the Azure PostgreSQL service. + Then restart your application. + """) + st.error(traceback.format_exc()) + else: + if llm_helper.vector_store.check_existing_index("embeddings-index"): + st.warning("""Seems like you're using a Redis with an old data structure. + If you want to use the new data structure, you can start using the app and go to "Add Document" -> "Add documents in Batch" and click on "Convert all files and add embeddings" to reprocess your documents. + To remove this working, please delete the index "embeddings-index" from your Redis. + If you prefer to use the old data structure, please change your Web App container image to point to the docker image: fruocco/oai-embeddings:2023-03-27_25. + """) + else: + st.success("Redis is working!") except Exception as e: st.error(f"""Redis is not working. Please check your Redis connection string in the App Settings. diff --git a/code/requirements.txt b/code/requirements.txt index 9ab217b..89571bb 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -19,5 +19,7 @@ beautifulsoup4==4.12.0 streamlit-chat==0.0.2.2 fake-useragent==1.1.3 chardet==5.1.0 +pgvector==0.2.4 +psycopg2-binary==2.9.9 --extra-index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/ azure-search-documents==11.4.0a20230509004 \ No newline at end of file diff --git a/code/utilities/azureblobstorage.py b/code/utilities/azureblobstorage.py index 2daa7d1..a73d4c1 100644 --- a/code/utilities/azureblobstorage.py +++ b/code/utilities/azureblobstorage.py @@ -8,9 +8,11 @@ def __init__(self, account_name: str = None, account_key: str = None, container_ load_dotenv() + self.azure_cloud : str = os.getenv('AZURE_CLOUD', 'AzureCloud') + self.blob_endpoint_suffix : str = 'core.chinacloudapi.cn' if self.azure_cloud == 'AzureChinaCloud' else 'core.windows.net' self.account_name : str = account_name if account_name else os.getenv('BLOB_ACCOUNT_NAME') self.account_key : str = account_key if account_key else os.getenv('BLOB_ACCOUNT_KEY') - self.connect_str : str = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};EndpointSuffix=core.windows.net" + self.connect_str : str = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};EndpointSuffix={self.blob_endpoint_suffix}" self.container_name : str = container_name if container_name else os.getenv('BLOB_CONTAINER_NAME') self.blob_service_client : BlobServiceClient = BlobServiceClient.from_connection_string(self.connect_str) @@ -40,12 +42,12 @@ def get_all_files(self): "filename" : blob.name, "converted": blob.metadata.get('converted', 'false') == 'true' if blob.metadata else False, "embeddings_added": blob.metadata.get('embeddings_added', 'false') == 'true' if blob.metadata else False, - "fullpath": f"https://{self.account_name}.blob.core.windows.net/{self.container_name}/{blob.name}?{sas}", + "fullpath": f"https://{self.account_name}.blob.{self.blob_endpoint_suffix}/{self.container_name}/{blob.name}?{sas}", "converted_filename": blob.metadata.get('converted_filename', '') if blob.metadata else '', "converted_path": "" }) else: - converted_files[blob.name] = f"https://{self.account_name}.blob.core.windows.net/{self.container_name}/{blob.name}?{sas}" + converted_files[blob.name] = f"https://{self.account_name}.blob.{self.blob_endpoint_suffix}/{self.container_name}/{blob.name}?{sas}" for file in files: converted_filename = file.pop('converted_filename', '') @@ -70,4 +72,4 @@ def get_container_sas(self): def get_blob_sas(self, file_name): # Generate a SAS URL to the blob and return it - return f"https://{self.account_name}.blob.core.windows.net/{self.container_name}/{file_name}" + "?" + generate_blob_sas(account_name= self.account_name, container_name=self.container_name, blob_name= file_name, account_key= self.account_key, permission='r', expiry=datetime.utcnow() + timedelta(hours=1)) + return f"https://{self.account_name}.blob.{self.blob_endpoint_suffix}/{self.container_name}/{file_name}" + "?" + generate_blob_sas(account_name= self.account_name, container_name=self.container_name, blob_name= file_name, account_key= self.account_key, permission='r', expiry=datetime.utcnow() + timedelta(hours=1)) diff --git a/code/utilities/helper.py b/code/utilities/helper.py index 8023556..1c6b6d0 100644 --- a/code/utilities/helper.py +++ b/code/utilities/helper.py @@ -28,6 +28,7 @@ from utilities.customprompt import PROMPT from utilities.redis import RedisExtended from utilities.azuresearch import AzureSearch +from utilities.pgvector import PGVectorExtended import pandas as pd import urllib @@ -69,10 +70,24 @@ def __init__(self, self.vector_store_type = os.getenv("VECTOR_STORE_TYPE") # Azure Search settings - if self.vector_store_type == "AzureSearch": + if self.vector_store_type == "AzureSearch": self.vector_store_address: str = os.getenv('AZURE_SEARCH_SERVICE_NAME') self.vector_store_password: str = os.getenv('AZURE_SEARCH_ADMIN_KEY') + # PGVector settings + elif self.vector_store_type == "PGVector": + self.vector_store_driver: str = os.getenv('PGVECTOR_DRIVER', "psycopg2") + self.vector_store_address: str = os.getenv('PGVECTOR_HOST', "localhost") + self.vector_store_port: int = int(os.getenv('PGVECTOR_PORT', 5432)) + self.vector_store_database: str = os.getenv("PGVECTOR_DATABASE", "postgres") + self.vector_store_username: str = os.getenv("PGVECTOR_USER", "postgres") + self.vector_store_password: str = os.getenv("PGVECTOR_PASSWORD", "postgres") + + if self.vector_store_password: + self.vector_store_full_address = f"postgresql+{self.vector_store_driver}://{self.vector_store_username}:{self.vector_store_password}@{self.vector_store_address}:{self.vector_store_port}/{self.vector_store_database}" + else: + self.vector_store_full_address = f"postgresql+{self.vector_store_driver}://{self.vector_store_username}@{self.vector_store_address}:{self.vector_store_port}/{self.vector_store_database}" + else: # Vector store settings self.vector_store_address: str = os.getenv('REDIS_ADDRESS', "localhost") @@ -94,8 +109,11 @@ def __init__(self, self.llm: ChatOpenAI = ChatOpenAI(model_name=self.deployment_name, engine=self.deployment_name, temperature=self.temperature, max_tokens=self.max_tokens if self.max_tokens != -1 else None) if llm is None else llm else: self.llm: AzureOpenAI = AzureOpenAI(deployment_name=self.deployment_name, temperature=self.temperature, max_tokens=self.max_tokens) if llm is None else llm + if self.vector_store_type == "AzureSearch": self.vector_store: VectorStore = AzureSearch(azure_cognitive_search_name=self.vector_store_address, azure_cognitive_search_key=self.vector_store_password, index_name=self.index_name, embedding_function=self.embeddings.embed_query) if vector_store is None else vector_store + elif self.vector_store_type == "PGVector": + self.vector_store: PGVectorExtended = PGVectorExtended(connection_string=self.vector_store_full_address, embedding_function=self.embeddings, collection_name="qnacollection", pre_delete_collection=False) if vector_store is None else vector_store else: self.vector_store: RedisExtended = RedisExtended(redis_url=self.vector_store_full_address, index_name=self.index_name, embedding_function=self.embeddings.embed_query) if vector_store is None else vector_store self.k : int = 3 if k is None else k @@ -138,8 +156,10 @@ def add_embeddings_lc(self, source_url): hash_key = f"doc:{self.index_name}:{hash_key}" keys.append(hash_key) doc.metadata = {"source": f"[{source_url}]({source_url}_SAS_TOKEN_PLACEHOLDER_)" , "chunk": i, "key": hash_key, "filename": filename} - if self.vector_store_type == 'AzureSearch': + if self.vector_store_type == "AzureSearch": self.vector_store.add_documents(documents=docs, keys=keys) + elif self.vector_store_type == "PGVector": + self.vector_store.add_documents(documents=docs, keys=keys, ids=keys) else: self.vector_store.add_documents(documents=docs, redis_url=self.vector_store_full_address, index_name=self.index_name, keys=keys) diff --git a/code/utilities/pgvector.py b/code/utilities/pgvector.py new file mode 100644 index 0000000..afd7236 --- /dev/null +++ b/code/utilities/pgvector.py @@ -0,0 +1,478 @@ +import enum +import logging +import uuid +from typing import Any, Dict, Iterable, List, Optional, Tuple + +import sqlalchemy +from sqlalchemy import delete +from pgvector.sqlalchemy import Vector +from sqlalchemy.dialects.postgresql import JSON, UUID +from sqlalchemy.orm import Mapped, Session, declarative_base, relationship + +from langchain.docstore.document import Document +from langchain.embeddings.base import Embeddings +from langchain.utils import get_from_dict_or_env +from langchain.vectorstores.base import VectorStore + +Base = declarative_base() # type: Any + + +ADA_TOKEN_COUNT = 1536 +_LANGCHAIN_DEFAULT_COLLECTION_NAME = "langchain" + + +class BaseModel(Base): + __abstract__ = True + uuid = sqlalchemy.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + +class CollectionStore(BaseModel): + __tablename__ = "langchain_pg_collection" + + name = sqlalchemy.Column(sqlalchemy.String) + cmetadata = sqlalchemy.Column(JSON) + + embeddings = relationship( + "EmbeddingStore", + back_populates="collection", + passive_deletes=True, + ) + + @classmethod + def get_by_name(cls, session: Session, name: str) -> Optional["CollectionStore"]: + return session.query(cls).filter(cls.name == name).first() + + @classmethod + def get_or_create( + cls, + session: Session, + name: str, + cmetadata: Optional[dict] = None, + ) -> Tuple["CollectionStore", bool]: + """ + Get or create a collection. + Returns [Collection, bool] where the bool is True if the collection was created. + """ + created = False + collection = cls.get_by_name(session, name) + if collection: + return collection, created + + collection = cls(name=name, cmetadata=cmetadata) + session.add(collection) + session.commit() + created = True + return collection, created + + +class EmbeddingStore(BaseModel): + __tablename__ = "langchain_pg_embedding" + + collection_id: Mapped[UUID] = sqlalchemy.Column( + UUID(as_uuid=True), + sqlalchemy.ForeignKey( + f"{CollectionStore.__tablename__}.uuid", + ondelete="CASCADE", + ), + ) + collection = relationship(CollectionStore, back_populates="embeddings") + + embedding: Vector = sqlalchemy.Column(Vector(ADA_TOKEN_COUNT)) + document = sqlalchemy.Column(sqlalchemy.String, nullable=True) + cmetadata = sqlalchemy.Column(JSON, nullable=True) + + # custom_id : any user defined id + custom_id = sqlalchemy.Column(sqlalchemy.String, nullable=True) + + +class QueryResult: + EmbeddingStore: EmbeddingStore + distance: float + + +class DistanceStrategy(str, enum.Enum): + EUCLIDEAN = EmbeddingStore.embedding.l2_distance + COSINE = EmbeddingStore.embedding.cosine_distance + MAX_INNER_PRODUCT = EmbeddingStore.embedding.max_inner_product + + +DEFAULT_DISTANCE_STRATEGY = DistanceStrategy.EUCLIDEAN + + +class PGVectorExtended(VectorStore): + """ + VectorStore implementation using Postgres and pgvector. + - `connection_string` is a postgres connection string. + - `embedding_function` any embedding function implementing + `langchain.embeddings.base.Embeddings` interface. + - `collection_name` is the name of the collection to use. (default: langchain) + - NOTE: This is not the name of the table, but the name of the collection. + The tables will be created when initializing the store (if not exists) + So, make sure the user has the right permissions to create tables. + - `distance_strategy` is the distance strategy to use. (default: EUCLIDEAN) + - `EUCLIDEAN` is the euclidean distance. + - `COSINE` is the cosine distance. + - `pre_delete_collection` if True, will delete the collection if it exists. + (default: False) + - Useful for testing. + """ + + def __init__( + self, + connection_string: str, + embedding_function: Embeddings, + collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME, + collection_metadata: Optional[dict] = None, + distance_strategy: DistanceStrategy = DEFAULT_DISTANCE_STRATEGY, + pre_delete_collection: bool = False, + logger: Optional[logging.Logger]= None, + engine_args: Optional[dict[str, Any]] = None, + ) -> None: + self.connection_string = connection_string + self.embedding_function = embedding_function + self.collection_name = collection_name + self.collection_metadata = collection_metadata + self.distance_strategy = distance_strategy + self.pre_delete_collection = pre_delete_collection + self.logger = logger or logging.getLogger(__name__) + self.engine_args = engine_args or {} + self._engine = self.connect() + # self._conn = self.connect() + # self.__post_init__() + self.CollectionStore = CollectionStore + self.EmbeddingStore = EmbeddingStore + + def __post_init__( + self, + ) -> None: + self.create_vector_extension() + self.create_tables_if_not_exists() + self.create_collection() + + def connect(self) -> sqlalchemy.engine: + engine = sqlalchemy.create_engine(self.connection_string, **self.engine_args) + return engine + + def create_vector_extension(self) -> None: + try: + with Session(self._engine) as session: + # The advisor lock fixes issue arising from concurrent + # creation of the vector extension. + # https://github.com/langchain-ai/langchain/issues/12933 + # For more information see: + # https://www.postgresql.org/docs/16/explicit-locking.html#ADVISORY-LOCKS + statement = sqlalchemy.text( + "BEGIN;" + "SELECT pg_advisory_xact_lock(1573678846307946496);" + "CREATE EXTENSION IF NOT EXISTS vector;" + "COMMIT;" + ) + session.execute(statement) + session.commit() + except Exception as e: + raise Exception(f"Failed to create vector extension: {e}") from e + + def create_tables_if_not_exists(self) -> None: + with self._engine.begin(): + Base.metadata.create_all(self._engine) + + def drop_tables(self) -> None: + with self._engine.begin(): + Base.metadata.drop_all(self._engine) + + def create_collection(self) -> None: + if self.pre_delete_collection: + self.delete_collection() + with Session(self._engine) as session: + CollectionStore.get_or_create( + session, self.collection_name, cmetadata=self.collection_metadata + ) + + def delete_collection(self) -> None: + self.logger.debug("Trying to delete collection") + with Session(self._engine) as session: + collection = self.get_collection(session) + if not collection: + self.logger.error("Collection not found") + return + session.delete(collection) + session.commit() + + def get_collection(self, session: Session) -> Optional["CollectionStore"]: + return CollectionStore.get_by_name(session, self.collection_name) + + def add_texts( + self, + texts: Iterable[str], + metadatas: Optional[List[dict]] = None, + ids: Optional[List[str]] = None, + **kwargs: Any, + ) -> List[str]: + """Run more texts through the embeddings and add to the vectorstore. + + Args: + texts: Iterable of strings to add to the vectorstore. + metadatas: Optional list of metadatas associated with the texts. + kwargs: vectorstore specific parameters + + Returns: + List of ids from adding the texts into the vectorstore. + """ + if ids is None: + ids = [str(uuid.uuid1()) for _ in texts] + + embeddings = self.embedding_function.embed_documents(list(texts)) + + if not metadatas: + metadatas = [{} for _ in texts] + + with Session(self._engine) as session: + collection = self.get_collection(session) + if not collection: + raise ValueError("Collection not found") + for text, metadata, embedding, id in zip(texts, metadatas, embeddings, ids): + embedding_store = EmbeddingStore( + embedding=embedding, + document=text, + cmetadata=metadata, + custom_id=id, + ) + collection.embeddings.append(embedding_store) + session.add(embedding_store) + session.commit() + + return ids + + def similarity_search( + self, + query: str, + k: int = 4, + filter: Optional[dict] = None, + **kwargs: Any, + ) -> List[Document]: + """Run similarity search with PGVector with distance. + + Args: + query (str): Query text to search for. + k (int): Number of results to return. Defaults to 4. + filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. + + Returns: + List of Documents most similar to the query. + """ + embedding = self.embedding_function.embed_query(text=query) + return self.similarity_search_by_vector( + embedding=embedding, + k=k, + filter=filter, + ) + + def similarity_search_with_score( + self, + query: str, + k: int = 4, + filter: Optional[dict] = None, + ) -> List[Tuple[Document, float]]: + """Return docs most similar to query. + + Args: + query: Text to look up documents similar to. + k: Number of Documents to return. Defaults to 4. + filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. + + Returns: + List of Documents most similar to the query and score for each + """ + embedding = self.embedding_function.embed_query(query) + docs = self.similarity_search_with_score_by_vector( + embedding=embedding, k=k, filter=filter + ) + return docs + + def similarity_search_with_score_by_vector( + self, + embedding: List[float], + k: int = 4, + filter: Optional[dict] = None, + ) -> List[Tuple[Document, float]]: + with Session(self._engine) as session: + collection = self.get_collection(session) + if not collection: + raise ValueError("Collection not found") + + filter_by = EmbeddingStore.collection_id == collection.uuid + + if filter is not None: + filter_clauses = [] + for key, value in filter.items(): + filter_by_metadata = EmbeddingStore.cmetadata[key].astext == str(value) + filter_clauses.append(filter_by_metadata) + + filter_by = sqlalchemy.and_(filter_by, *filter_clauses) + + results: List[QueryResult] = ( + session.query( + EmbeddingStore, + self.distance_strategy(embedding).label("distance"), # type: ignore + ) + .filter(filter_by) + .order_by(sqlalchemy.asc("distance")) + .join( + CollectionStore, + EmbeddingStore.collection_id == CollectionStore.uuid, + ) + .limit(k) + .all() + ) + docs = [ + ( + Document( + page_content=result.EmbeddingStore.document, + metadata=result.EmbeddingStore.cmetadata, + ), + result.distance if self.embedding_function is not None else None, + ) + for result in results + ] + return docs + + def similarity_search_by_vector( + self, + embedding: List[float], + k: int = 4, + filter: Optional[dict] = None, + **kwargs: Any, + ) -> List[Document]: + """Return docs most similar to embedding vector. + + Args: + embedding: Embedding to look up documents similar to. + k: Number of Documents to return. Defaults to 4. + filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. + + Returns: + List of Documents most similar to the query vector. + """ + docs_and_scores = self.similarity_search_with_score_by_vector( + embedding=embedding, k=k, filter=filter + ) + return [doc for doc, _ in docs_and_scores] + + @classmethod + def from_texts( + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME, + distance_strategy: DistanceStrategy = DistanceStrategy.COSINE, + ids: Optional[List[str]] = None, + pre_delete_collection: bool = False, + **kwargs: Any, + ) -> "PGVectorExtended": + """ + Return VectorStore initialized from texts and embeddings. + Postgres connection string is required + "Either pass it as a parameter + or set the PGVECTOR_CONNECTION_STRING environment variable. + """ + + connection_string = cls.get_connection_string(kwargs) + + store = cls( + connection_string=connection_string, + collection_name=collection_name, + embedding_function=embedding, + distance_strategy=distance_strategy, + pre_delete_collection=pre_delete_collection, + ) + + store.add_texts(texts=texts, metadatas=metadatas, ids=ids, **kwargs) + return store + + @classmethod + def get_connection_string(cls, kwargs: Dict[str, Any]) -> str: + connection_string: str = get_from_dict_or_env( + data=kwargs, + key="connection_string", + env_key="PGVECTOR_CONNECTION_STRING", + ) + + if not connection_string: + raise ValueError( + "Postgres connection string is required" + "Either pass it as a parameter" + "or set the PGVECTOR_CONNECTION_STRING environment variable." + ) + + return connection_string + + @classmethod + def from_documents( + cls, + documents: List[Document], + embedding: Embeddings, + collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME, + distance_strategy: DistanceStrategy = DEFAULT_DISTANCE_STRATEGY, + ids: Optional[List[str]] = None, + pre_delete_collection: bool = False, + **kwargs: Any, + ) -> "PGVectorExtended": + """ + Return VectorStore initialized from documents and embeddings. + Postgres connection string is required + "Either pass it as a parameter + or set the PGVECTOR_CONNECTION_STRING environment variable. + """ + + texts = [d.page_content for d in documents] + metadatas = [d.metadata for d in documents] + connection_string = cls.get_connection_string(kwargs) + + kwargs["connection_string"] = connection_string + + return cls.from_texts( + texts=texts, + pre_delete_collection=pre_delete_collection, + embedding=embedding, + distance_strategy=distance_strategy, + metadatas=metadatas, + ids=ids, + collection_name=collection_name, + **kwargs, + ) + + @classmethod + def connection_string_from_db_params( + cls, + driver: str, + host: str, + port: int, + database: str, + user: str, + password: str, + ) -> str: + """Return connection string from database parameters.""" + return f"postgresql+{driver}://{user}:{password}@{host}:{port}/{database}" + + def delete_keys( + self, + ids: Optional[List[str]] = None, + **kwargs: Any, + ) -> None: + """Delete vectors by ids or uuids. + + Args: + ids: List of ids to delete. + """ + with Session(self._engine) as session: + if ids is not None: + self.logger.debug( + "Trying to delete vectors by ids (represented by the model " + "using the custom ids field)" + ) + stmt = delete(self.EmbeddingStore).where( + self.EmbeddingStore.custom_id.in_(ids) + ) + session.execute(stmt) + session.commit() \ No newline at end of file diff --git a/docs/architecture_pg.png b/docs/architecture_pg.png new file mode 100644 index 0000000..43628c9 Binary files /dev/null and b/docs/architecture_pg.png differ diff --git a/infrastructure/deployment_azcn.json b/infrastructure/deployment_azcn.json new file mode 100644 index 0000000..42038e4 --- /dev/null +++ b/infrastructure/deployment_azcn.json @@ -0,0 +1,562 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourcePrefix": { + "type": "string", + "metadata": { + "description": "provide a 2-13 character prefix for all resources. Be sure to lowercase." + } + }, + "ContainerName": { + "type": "string", + "defaultValue": "redis", + "metadata": { + "description": "Name of the container" + } + }, + "DNSNameLabel": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-redis')]", + "metadata": { + "description": "DNS Name Label for the Public IP address" + } + }, + "RedisPassword": { + "type": "securestring", + "defaultValue": "redis", + "metadata": { + "description": "Redis Password" + } + }, + "HostingPlanName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-plan')]", + "metadata": { + "description": "Name of App Service plan" + } + }, + "HostingPlanSku": { + "type": "string", + "defaultValue": "P3V2", + "allowedValues": [ + "B3", + "S2", + "S3", + "P2V2", + "P3V2" + ], + "metadata": { + "description": "The pricing tier for the App Service plan" + } + }, + "StorageAccountName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), 'str')]", + "metadata": { + "description": "Name of Storage Account" + } + }, + "WebsiteName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-site')]", + "metadata": { + "description": "Name of Web App" + } + }, + "FunctionName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-batchfunc')]", + "metadata": { + "description": "Name of Function App for Batch document processing" + } + }, + "ApplicationInsightsName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-appinsights')]", + "metadata": { + "description": "Name of Application Insights" + } + }, + "OpenAIName": { + "type": "string", + "metadata": { + "description": "Name of OpenAI Resource" + } + }, + "OpenAIKey": { + "type": "securestring", + "metadata": { + "description": "OpenAI API Key" + } + }, + "OpenAIEngine": { + "type": "string", + "metadata": { + "description": "OpenAI Engine" + } + }, + "OpenAIDeploymentType": { + "type": "string", + "allowedValues": [ + "Chat", + "Text" + ], + "metadata": { + "description": "OpenAI Deployment Type. Text for an Instructions based deployment (text-davinci-003). Chat for a Chat based deployment (gpt-35-turbo or gpt-4-32k or gpt-4)." + } + }, + "OpenAIEmbeddingsEngineDoc": { + "type": "string", + "defaultValue": "text-embedding-ada-002", + "metadata": { + "description": "OpenAI Embeddings Engine for Documents" + } + }, + "OpenAIEmbeddingsEngineQuery": { + "type": "string", + "defaultValue": "text-embedding-ada-002", + "metadata": { + "description": "OpenAI Embeddings Engine for Queries" + } + }, + "OpenAITemperature": { + "type": "string", + "defaultValue": "0.1", + "metadata": { + "description": "OpenAI Temperature" + } + }, + "OpenAIMaxTokens": { + "type": "string", + "defaultValue": "-1", + "metadata": { + "description": "OpenAI Max Tokens" + } + }, + "DocumentIntelligenceEndpoint": { + "type": "string", + "metadata": { + "description": "Document Intelligence Endpoint" + } + }, + "DocumentIntelligenceKey": { + "type": "securestring", + "metadata": { + "description": "Document Intelligence Key" + } + }, + "TranslateEndpoint": { + "type": "string", + "defaultValue": "https://api.translator.azure.cn/", + "metadata": { + "description": "Translator Endpoint" + } + }, + "TranslateKey": { + "type": "securestring", + "metadata": { + "description": "Translator Key" + } + }, + "TranslateRegion": { + "type": "string", + "allowedValues": [ + "chinaeast2", + "chinanorth", + "chinanorth2", + "chinanorth3" + ], + "metadata": { + "description": "Translator Region" + } + }, + "newGuid": { + "type": "string", + "defaultValue": "[newGuid()]" + } + }, + "variables": { + "BlobContainerName": "documents", + "FileShareName": "redisdata", + "QueueName": "doc-processing", + "ClientKey": "[concat(uniqueString(guid(resourceGroup().id, deployment().name)), parameters('newGuid'), 'Tg2%')]" + }, + "resources": [ + { + "apiVersion": "2020-06-01", + "name": "[parameters('HostingPlanName')]", + "type": "Microsoft.Web/serverfarms", + "location": "[resourceGroup().location]", + "sku": { + "name": "[parameters('HostingPlanSku')]" + }, + "properties": { + "name": "[parameters('HostingPlanName')]", + "reserved": true + }, + "kind": "linux" + }, + { + "apiVersion": "2020-06-01", + "name": "[parameters('WebsiteName')]", + "type": "Microsoft.Web/sites", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', parameters('HostingPlanName'))]" + ], + "properties": { + "serverFarmId": "[parameters('HostingPlanName')]", + "siteConfig": { + "linuxFxVersion": "DOCKER|cyberflying/aoai-web:latest" + } + } + }, + { + "name": "[parameters('StorageAccountName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-08-01", + "location": "[resourceGroup().location]", + "kind": "StorageV2", + "sku": { + "name": "Standard_GRS" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2021-08-01", + "name": "[concat(parameters('StorageAccountName'), '/default/', variables('BlobContainerName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2021-08-01", + "name": "[concat(parameters('StorageAccountName'), '/default/', variables('FileShareName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + + "protocolSettings": { + "smb": {} + }, + "cors": { + "corsRules": [] + }, + "shareDeleteRetentionPolicy": { + "enabled": true, + "days": 7 + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "cors": { + "corsRules": [] + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices/queues", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default/doc-processing')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', parameters('StorageAccountName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "metadata": {} + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices/queues", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default/doc-processing-poison')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', parameters('StorageAccountName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "metadata": {} + } + }, + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[parameters('ApplicationInsightsName')]", + "location": "[resourceGroup().location]", + "tags": { + "[concat('hidden-link:', resourceId('Microsoft.Web/sites', parameters('ApplicationInsightsName')))]": "Resource" + }, + "properties": { + "Application_Type": "web" + }, + "kind": "web" + }, + { + "apiVersion": "2018-11-01", + "name": "[parameters('FunctionName')]", + "type": "Microsoft.Web/sites", + "kind": "functionapp,linux", + "location": "[resourceGroup().location]", + "tags": {}, + "dependsOn": [ + "[concat('Microsoft.Web/serverfarms/', parameters('HostingPlanName'))]", + "[concat('Microsoft.Storage/storageAccounts/', parameters('StorageAccountName'))]", + "[concat('Microsoft.Insights/components/', parameters('ApplicationInsightsName'))]" + ], + "properties": { + "name": "[parameters('FunctionName')]", + "siteConfig": { + "appSettings": [ + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "WEBSITES_ENABLE_APP_SERVICE_STORAGE", + "value": "false" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', parameters('ApplicationInsightsName')), '2015-05-01').InstrumentationKey]" + }, + { + "name": "AzureWebJobsStorage", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('StorageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.chinacloudapi.cn')]" + }, + { + "name": "OPENAI_ENGINE", + "value": "[parameters('OpenAIEngine')]" + }, + { + "name": "OPENAI_DEPLOYMENT_TYPE", + "value": "[parameters('OpenAIDeploymentType')]" + }, + { + "name": "OPENAI_EMBEDDINGS_ENGINE_DOC", + "value": "[parameters('OpenAIEmbeddingsEngineDoc')]" + }, + { + "name": "OPENAI_EMBEDDINGS_ENGINE_QUERY", + "value": "[parameters('OpenAIEmbeddingsEngineQuery')]" + }, + { + "name": "OPENAI_API_BASE", + "value": "[concat('https://', parameters('OpenAIName'), '.openai.azure.com/')]" + }, + { + "name": "OPENAI_API_KEY", + "value": "[parameters('OpenAIKey')]" + }, + { + "name": "OPENAI_TEMPERATURE", + "value": "[parameters('OpenAITemperature')]" + }, + { + "name": "OPENAI_MAX_TOKENS", + "value": "[parameters('OpenAIMaxTokens')]" + }, + { + "name": "BLOB_ACCOUNT_NAME", + "value": "[parameters('StorageAccountName')]" + }, + { + "name": "BLOB_ACCOUNT_KEY", + "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value]" + }, + { + "name": "BLOB_CONTAINER_NAME", + "value": "[variables('BlobContainerName')]" + }, + { + "name": "AZURE_CLOUD", + "value": "AzureChinaCloud" + }, + { + "name": "FORM_RECOGNIZER_ENDPOINT", + "value": "[parameters('DocumentIntelligenceEndpoint')]" + }, + { + "name": "FORM_RECOGNIZER_KEY", + "value": "[parameters('DocumentIntelligenceKey')]" + }, + { + "name": "REDIS_ADDRESS", + "value": "[concat(parameters('DNSNameLabel'),'.' , resourceGroup().location ,'.azurecontainer.console.azure.cn')]" + }, + { + "name": "REDIS_PASSWORD", + "value": "[parameters('RedisPassword')]" + }, + { + "name": "TRANSLATE_ENDPOINT", + "value": "[parameters('TranslateEndpoint')]" + }, + { + "name": "TRANSLATE_KEY", + "value": "[parameters('TranslateKey')]" + }, + { + "name": "TRANSLATE_REGION", + "value": "[parameters('TranslateRegion')]" + }, + { + "name": "QUEUE_NAME", + "value": "[variables('QueueName')]" + } + ], + "cors": { + "allowedOrigins": [ + "https://portal.azure.com" + ] + }, + "use32BitWorkerProcess": false, + "linuxFxVersion": "DOCKER|cyberflying/aoai-batch:latest", + "appCommandLine": "", + "alwaysOn": true + }, + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('HostingPlanName'))]", + "clientAffinityEnabled": false, + "virtualNetworkSubnetId": null, + "httpsOnly": true + } + }, + { + "name": "[parameters('ContainerName')]", + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "2021-10-01", + "location": "[resourceGroup().location]", + "properties": { + "containers": [ + { + "name": "[parameters('ContainerName')]", + "properties": { + "image": "dockerhub.azk8s.cn/redis/redis-stack-server:latest", + "resources": { + "requests": { + "cpu": 4, + "memoryInGb": 16 + } + }, + "environmentVariables": [ + { + "name": "REDIS_ARGS", + "value": "[concat('--requirepass ', parameters('RedisPassword'))]" + } + ], + "ports": [ + { + "protocol": "TCP", + "port": 6379 + } + ], + "volumeMounts": [ + { + "name": "[variables('FileShareName')]", + "mountPath": "/data" + } + ] + } + } + ], + "osType": "Linux", + "ipAddress": { + "type": "Public", + "ports": [ + { + "protocol": "TCP", + "port": 6379 + } + ], + "dnsNameLabel": "[parameters('DNSNameLabel')]" + }, + "initContainers": [ + ], + "volumes": [ + { + "name": "[variables('FileShareName')]", + "azureFile": { + "shareName": "[variables('FileShareName')]", + "storageAccountName": "[parameters('StorageAccountName')]", + "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName')), '2021-08-01').keys[0].value]" + } + } + ] + } + }, + { + "type": "Microsoft.Web/sites/host/functionKeys", + "apiVersion": "2018-11-01", + "name": "[concat(parameters('FunctionName'), '/default/clientKey')]", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('FunctionName'))]", + "WaitFunctionDeploymentSection" + ], + "properties": { + "name": "ClientKey", + "value": "[variables('ClientKey')]" + } + }, + { + "type": "Microsoft.Web/sites/config", + "apiVersion": "2021-03-01", + "name": "[format('{0}/{1}', parameters('WebsiteName'), 'appsettings')]", + "kind": "string", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('WebsiteName'))]", + "[concat('Microsoft.Insights/components/', parameters('ApplicationInsightsName'))]" + ], + "properties": { + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.Insights/components', parameters('ApplicationInsightsName')), '2015-05-01').InstrumentationKey]", + "OPENAI_ENGINE": "[parameters('OpenAIEngine')]", + "OPENAI_DEPLOYMENT_TYPE": "[parameters('OpenAIDeploymentType')]", + "OPENAI_EMBEDDINGS_ENGINE_DOC": "[parameters('OpenAIEmbeddingsEngineDoc')]", + "OPENAI_EMBEDDINGS_ENGINE_QUERY": "[parameters('OpenAIEmbeddingsEngineQuery')]", + "AZURE_CLOUD": "AzureChinaCloud", + "REDIS_ADDRESS": "[concat(parameters('DNSNameLabel'),'.' , resourceGroup().location ,'.azurecontainer.console.azure.cn')]", + "REDIS_PASSWORD": "[parameters('RedisPassword')]", + "OPENAI_API_BASE": "[concat('https://', parameters('OpenAIName'),'.openai.azure.com/')]", + "OPENAI_API_KEY": "[parameters('OpenAIKey')]", + "OPENAI_TEMPERATURE": "[parameters('OpenAITemperature')]", + "OPENAI_MAX_TOKENS": "[parameters('OpenAIMaxTokens')]", + "BLOB_ACCOUNT_NAME": "[parameters('StorageAccountName')]", + "BLOB_ACCOUNT_KEY": "[listkeys(resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName')), '2015-05-01-preview').key1]", + "BLOB_CONTAINER_NAME": "[variables('BlobContainerName')]", + "FORM_RECOGNIZER_ENDPOINT": "[parameters('DocumentIntelligenceEndpoint')]", + "FORM_RECOGNIZER_KEY": "[parameters('DocumentIntelligenceKey')]", + "TRANSLATE_ENDPOINT": "[parameters('TranslateEndpoint')]", + "TRANSLATE_KEY": "[parameters('TranslateKey')]", + "TRANSLATE_REGION": "[parameters('TranslateRegion')]", + "CONVERT_ADD_EMBEDDINGS_URL": "[concat('https://', parameters('FunctionName') , '.chinacloudsites.cn/api/BatchStartProcessing?code=', variables('ClientKey'))]" + } + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "kind": "AzurePowerShell", + "name": "WaitFunctionDeploymentSection", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('FunctionName'))]" + ], + "properties": { + "azPowerShellVersion": "3.0", + "scriptContent": "start-sleep -Seconds 100", + "cleanupPreference": "Always", + "retentionInterval": "PT1H" + } + } + ] +} diff --git a/infrastructure/deployment_pg.json b/infrastructure/deployment_pg.json new file mode 100644 index 0000000..f6e1904 --- /dev/null +++ b/infrastructure/deployment_pg.json @@ -0,0 +1,643 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourcePrefix": { + "type": "string", + "metadata": { + "description": "provide a 2-13 character prefix for all resources. Be sure to lowercase." + } + }, + "AzureDatabaseForPostgreSQLName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-pgsql')]", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Name" + } + }, + "AzureDatabaseForPostgreSQLLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Location." + } + }, + "AzureDatabaseForPostgreSQLVersion": { + "type": "string", + "defaultValue": "16", + "allowedValues": [ + "15", + "16" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Version" + } + }, + "AzureDatabaseForPostgreSQLAdminUsername": { + "type": "string", + "defaultValue": "postgres", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Admin Username" + } + }, + "AzureDatabaseForPostgreSQLAdminPassword": { + "type": "securestring", + "defaultValue": "postgres", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Admin Password" + } + }, + "AzureDatabaseForPostgreSQLSkuTier": { + "type": "string", + "defaultValue": "GeneralPurpose", + "allowedValues": [ + "GeneralPurpose", + "MemoryOptimized" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server SKU Tier" + } + }, + "AzureDatabaseForPostgreSQLSkuName": { + "type": "string", + "defaultValue": "Standard_E64ds_v4", + "allowedValues": [ + "Standard_D4ds_v4", + "Standard_D8ds_v4", + "Standard_D16ds_v4", + "Standard_D32ds_v4", + "Standard_D48ds_v4", + "Standard_D64ds_v4", + "Standard_E8ds_v4", + "Standard_E16ds_v4", + "Standard_E32ds_v4", + "Standard_E48ds_v4", + "Standard_E64ds_v4" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server SKU Name" + } + }, + "AzureDatabaseForPostgreSQLStorageSize": { + "type": "int", + "defaultValue": 32768, + "allowedValues": [ + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768 + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Storage Size" + } + }, + "DocumentIntelligenceName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-doci')]", + "metadata": { + "description": "Document intelligence Name" + } + }, + "TranslatorName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-translator')]", + "metadata": { + "description": "Translator Name" + } + }, + "HostingPlanName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-plan')]", + "metadata": { + "description": "Name of App Service plan" + } + }, + "HostingPlanSku": { + "type": "string", + "defaultValue": "P3V2", + "allowedValues": [ + "B3", + "S2", + "S3", + "P2V2", + "P3V2" + ], + "metadata": { + "description": "The pricing tier for the App Service plan" + } + }, + "StorageAccountName": { + "type": "string", + "defaultValue": "[concat(toLower(parameters('ResourcePrefix')), 'str')]", + "metadata": { + "description": "Name of Storage Account" + } + }, + "WebsiteName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-site')]", + "metadata": { + "description": "Name of Web App" + } + }, + "FunctionName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-batchfunc')]", + "metadata": { + "description": "Name of Function App for Batch document processing" + } + }, + "ApplicationInsightsName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-appinsights')]", + "metadata": { + "description": "Name of Application Insights" + } + }, + "OpenAIName": { + "type": "string", + "metadata": { + "description": "Name of OpenAI Resource" + } + }, + "OpenAIKey": { + "type": "securestring", + "metadata": { + "description": "OpenAI API Key" + } + }, + "OpenAIEngine": { + "type": "string", + "metadata": { + "description": "OpenAI Engine" + } + }, + "OpenAIDeploymentType": { + "type": "string", + "allowedValues": [ + "Chat", + "Text" + ], + "metadata": { + "description": "OpenAI Deployment Type. Text for an Instructions based deployment (text-davinci-003). Chat for a Chat based deployment (gpt-35-turbo or gpt-4-32k or gpt-4)." + } + }, + "OpenAIEmbeddingsEngineDoc": { + "type": "string", + "defaultValue": "text-embedding-ada-002", + "metadata": { + "description": "OpenAI Embeddings Engine for Documents" + } + }, + "OpenAIEmbeddingsEngineQuery": { + "type": "string", + "defaultValue": "text-embedding-ada-002", + "metadata": { + "description": "OpenAI Embeddings Engine for Queries" + } + }, + "OpenAITemperature": { + "type": "string", + "defaultValue": "0.1", + "metadata": { + "description": "OpenAI Temperature" + } + }, + "OpenAIMaxTokens": { + "type": "string", + "defaultValue": "-1", + "metadata": { + "description": "OpenAI Max Tokens" + } + }, + "newGuid": { + "type": "string", + "defaultValue": "[newGuid()]" + } + }, + "variables": { + "VectorStoreType": "PGVector", + "BlobContainerName": "documents", + "QueueName": "doc-processing", + "ClientKey": "[concat(uniqueString(guid(resourceGroup().id, deployment().name)), parameters('newGuid'), 'Tg2%')]" + }, + "resources": [ + { + "type": "Microsoft.DBforPostgreSQL/flexibleServers", + "apiVersion": "2022-12-01", + "name": "[parameters('AzureDatabaseForPostgreSQLName')]", + "location": "[parameters('AzureDatabaseForPostgreSQLLocation')]", + "sku": { + "name": "[parameters('AzureDatabaseForPostgreSQLSkuName')]", + "tier": "[parameters('AzureDatabaseForPostgreSQLSkuTier')]" + }, + "properties": { + "administratorLogin": "[parameters('AzureDatabaseForPostgreSQLAdminUsername')]", + "administratorLoginPassword": "[parameters('AzureDatabaseForPostgreSQLAdminPassword')]", + "storage": { + "storageSizeGB": "[parameters('AzureDatabaseForPostgreSQLStorageSize')]" + }, + "createMode": "Default", + "version": "[parameters('AzureDatabaseForPostgreSQLVersion')]", + "backup": { + "backupRetentionDays": 35 + }, + "highAvailability": { + "mode": "Disabled" + }, + "publicNetworkAccess": "Enabled" + }, + "resources": [ + { + "type": "firewallRules", + "apiVersion": "2023-03-01-preview", + "name": "AllowAzureServices", + "dependsOn": [ + "[resourceId('Microsoft.DBforPostgreSQL/flexibleServers', parameters('AzureDatabaseForPostgreSQLName'))]" + ], + "properties": { + "startIpAddress": "0.0.0.0", + "endIpAddress": "0.0.0.0", + "startIpAddressRang": "AzureServices", + "endIpAddressRang": "AzureServices" + } + } + ] + }, + { + "type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations", + "apiVersion": "2022-12-01", + "name": "[concat(parameters('AzureDatabaseForPostgreSQLName'), '/azure.extensions')]", + "dependsOn": [ + "[resourceId('Microsoft.DBforPostgreSQL/flexibleServers', parameters('AzureDatabaseForPostgreSQLName'))]" + ], + "properties": { + "value": "vector", + "source": "user-override" + } + }, + { + "type": "Microsoft.CognitiveServices/accounts", + "apiVersion": "2022-12-01", + "name": "[parameters('DocumentIntelligenceName')]", + "location": "[resourceGroup().location]", + "sku": { + "name": "S0" + }, + "kind": "FormRecognizer", + "identity": { + "type": "None" + }, + "properties": { + "customSubDomainName": "[toLower(parameters('DocumentIntelligenceName'))]", + "networkAcls": { + "defaultAction": "Allow", + "virtualNetworkRules": [], + "ipRules": [] + }, + "publicNetworkAccess": "Enabled" + } + }, + { + "type": "Microsoft.CognitiveServices/accounts", + "apiVersion": "2022-12-01", + "name": "[parameters('TranslatorName')]", + "location": "[resourceGroup().location]", + "sku": { + "name": "S1" + }, + "kind": "TextTranslation", + "identity": { + "type": "None" + }, + "properties": { + "customSubDomainName": "[toLower(parameters('TranslatorName'))]", + "networkAcls": { + "defaultAction": "Allow", + "virtualNetworkRules": [], + "ipRules": [] + }, + "publicNetworkAccess": "Enabled" + } + }, + { + "apiVersion": "2020-06-01", + "name": "[parameters('HostingPlanName')]", + "type": "Microsoft.Web/serverfarms", + "location": "[resourceGroup().location]", + "sku": { + "name": "[parameters('HostingPlanSku')]" + }, + "properties": { + "name": "[parameters('HostingPlanName')]", + "reserved": true + }, + "kind": "linux" + }, + { + "apiVersion": "2020-06-01", + "name": "[parameters('WebsiteName')]", + "type": "Microsoft.Web/sites", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', parameters('HostingPlanName'))]" + ], + "properties": { + "serverFarmId": "[parameters('HostingPlanName')]", + "siteConfig": { + "linuxFxVersion": "DOCKER|cyberflying/aoai-web:latest" + } + } + }, + { + "name": "[parameters('StorageAccountName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-08-01", + "location": "[resourceGroup().location]", + "kind": "StorageV2", + "sku": { + "name": "Standard_GRS" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2021-08-01", + "name": "[concat(parameters('StorageAccountName'), '/default/', variables('BlobContainerName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "cors": { + "corsRules": [] + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices/queues", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default/doc-processing')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', parameters('StorageAccountName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "metadata": {} + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices/queues", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default/doc-processing-poison')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', parameters('StorageAccountName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "metadata": {} + } + }, + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[parameters('ApplicationInsightsName')]", + "location": "[resourceGroup().location]", + "tags": { + "[concat('hidden-link:', resourceId('Microsoft.Web/sites', parameters('ApplicationInsightsName')))]": "Resource" + }, + "properties": { + "Application_Type": "web" + }, + "kind": "web" + }, + { + "apiVersion": "2018-11-01", + "name": "[parameters('FunctionName')]", + "type": "Microsoft.Web/sites", + "kind": "functionapp,linux", + "location": "[resourceGroup().location]", + "tags": {}, + "dependsOn": [ + "[concat('Microsoft.Web/serverfarms/', parameters('HostingPlanName'))]", + "[concat('Microsoft.Storage/storageAccounts/', parameters('StorageAccountName'))]", + "[concat('Microsoft.Insights/components/', parameters('ApplicationInsightsName'))]" + ], + "properties": { + "name": "[parameters('FunctionName')]", + "siteConfig": { + "appSettings": [ + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "WEBSITES_ENABLE_APP_SERVICE_STORAGE", + "value": "false" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', parameters('ApplicationInsightsName')), '2015-05-01').InstrumentationKey]" + }, + { + "name": "AzureWebJobsStorage", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('StorageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.windows.net')]" + }, + { + "name": "OPENAI_ENGINE", + "value": "[parameters('OpenAIEngine')]" + }, + { + "name": "OPENAI_DEPLOYMENT_TYPE", + "value": "[parameters('OpenAIDeploymentType')]" + }, + { + "name": "OPENAI_EMBEDDINGS_ENGINE_DOC", + "value": "[parameters('OpenAIEmbeddingsEngineDoc')]" + }, + { + "name": "OPENAI_EMBEDDINGS_ENGINE_QUERY", + "value": "[parameters('OpenAIEmbeddingsEngineQuery')]" + }, + { + "name": "OPENAI_API_BASE", + "value": "[concat('https://', parameters('OpenAIName'), '.openai.azure.com/')]" + }, + { + "name": "OPENAI_API_KEY", + "value": "[parameters('OpenAIKey')]" + }, + { + "name": "OPENAI_TEMPERATURE", + "value": "[parameters('OpenAITemperature')]" + }, + { + "name": "OPENAI_MAX_TOKENS", + "value": "[parameters('OpenAIMaxTokens')]" + }, + { + "name": "BLOB_ACCOUNT_NAME", + "value": "[parameters('StorageAccountName')]" + }, + { + "name": "BLOB_ACCOUNT_KEY", + "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value]" + }, + { + "name": "BLOB_CONTAINER_NAME", + "value": "[variables('BlobContainerName')]" + }, + { + "name": "AZURE_CLOUD", + "value": "AzureCloud" + }, + { + "name": "VECTOR_STORE_TYPE", + "value": "[variables('VectorStoreType')]" + }, + { + "name": "PGVECTOR_DRIVER", + "value": "psycopg2" + }, + { + "name": "PGVECTOR_HOST", + "value": "[concat(parameters('AzureDatabaseForPostgreSQLName'), '.postgres.database.azure.com')]" + }, + { + "name": "PGVECTOR_PORT", + "value": "5432" + }, + { + "name": "PGVECTOR_DATABASE", + "value": "postgres" + }, + { + "name": "PGVECTOR_USER", + "value": "[parameters('AzureDatabaseForPostgreSQLAdminUsername')]" + }, + { + "name": "PGVECTOR_PASSWORD", + "value": "[parameters('AzureDatabaseForPostgreSQLAdminPassword')]" + }, + { + "name": "FORM_RECOGNIZER_ENDPOINT", + "value": "[concat('https://',parameters('DocumentIntelligenceName'),'.cognitiveservices.azure.com/')]" + }, + { + "name": "FORM_RECOGNIZER_KEY", + "value": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('DocumentIntelligenceName')), '2023-05-01').key1]" + }, + { + "name": "TRANSLATE_ENDPOINT", + "value": "https://api.cognitive.microsofttranslator.com/" + }, + { + "name": "TRANSLATE_KEY", + "value": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('TranslatorName')), '2023-05-01').key1]" + }, + { + "name": "TRANSLATE_REGION", + "value": "[resourceGroup().location]" + }, + { + "name": "QUEUE_NAME", + "value": "[variables('QueueName')]" + } + ], + "cors": { + "allowedOrigins": [ + "https://portal.azure.com" + ] + }, + "use32BitWorkerProcess": false, + "linuxFxVersion": "DOCKER|cyberflying/aoai-batch:latest", + "appCommandLine": "", + "alwaysOn": true + }, + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('HostingPlanName'))]", + "clientAffinityEnabled": false, + "virtualNetworkSubnetId": null, + "httpsOnly": true + } + }, + { + "type": "Microsoft.Web/sites/host/functionKeys", + "apiVersion": "2018-11-01", + "name": "[concat(parameters('FunctionName'), '/default/clientKey')]", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('FunctionName'))]", + "WaitFunctionDeploymentSection" + ], + "properties": { + "name": "ClientKey", + "value": "[variables('ClientKey')]" + } + }, + { + "type": "Microsoft.Web/sites/config", + "apiVersion": "2021-03-01", + "name": "[format('{0}/{1}', parameters('WebsiteName'), 'appsettings')]", + "kind": "string", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('WebsiteName'))]", + "[concat('Microsoft.Insights/components/', parameters('ApplicationInsightsName'))]" + ], + "properties": { + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.Insights/components', parameters('ApplicationInsightsName')), '2015-05-01').InstrumentationKey]", + "OPENAI_ENGINE": "[parameters('OpenAIEngine')]", + "OPENAI_DEPLOYMENT_TYPE": "[parameters('OpenAIDeploymentType')]", + "OPENAI_EMBEDDINGS_ENGINE_DOC": "[parameters('OpenAIEmbeddingsEngineDoc')]", + "OPENAI_EMBEDDINGS_ENGINE_QUERY": "[parameters('OpenAIEmbeddingsEngineQuery')]", + "AZURE_CLOUD": "AzureCloud", + "VECTOR_STORE_TYPE": "[variables('VectorStoreType')]", + "PGVECTOR_DRIVER": "psycopg2", + "PGVECTOR_HOST": "[concat(parameters('AzureDatabaseForPostgreSQLName'), '.postgres.database.azure.com')]", + "PGVECTOR_PORT": "5432", + "PGVECTOR_DATABASE": "postgres", + "PGVECTOR_USER": "[parameters('AzureDatabaseForPostgreSQLAdminUsername')]", + "PGVECTOR_PASSWORD": "[parameters('AzureDatabaseForPostgreSQLAdminPassword')]", + "OPENAI_API_BASE": "[concat('https://', parameters('OpenAIName'),'.openai.azure.com/')]", + "OPENAI_API_KEY": "[parameters('OpenAIKey')]", + "OPENAI_TEMPERATURE": "[parameters('OpenAITemperature')]", + "OPENAI_MAX_TOKENS": "[parameters('OpenAIMaxTokens')]", + "BLOB_ACCOUNT_NAME": "[parameters('StorageAccountName')]", + "BLOB_ACCOUNT_KEY": "[listkeys(resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName')), '2015-05-01-preview').key1]", + "BLOB_CONTAINER_NAME": "[variables('BlobContainerName')]", + "FORM_RECOGNIZER_ENDPOINT": "[concat('https://',parameters('DocumentIntelligenceName'),'.cognitiveservices.azure.com/')]", + "FORM_RECOGNIZER_KEY": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('DocumentIntelligenceName')), '2023-05-01').key1]", + "TRANSLATE_ENDPOINT": "https://api.cognitive.microsofttranslator.com/", + "TRANSLATE_KEY": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('TranslatorName')), '2023-05-01').key1]", + "TRANSLATE_REGION": "[resourceGroup().location]", + "CONVERT_ADD_EMBEDDINGS_URL": "[concat('https://', parameters('FunctionName') , '.azurewebsites.net/api/BatchStartProcessing?code=', variables('ClientKey'))]" + } + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "kind": "AzurePowerShell", + "name": "WaitFunctionDeploymentSection", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('FunctionName'))]" + ], + "properties": { + "azPowerShellVersion": "3.0", + "scriptContent": "start-sleep -Seconds 100", + "cleanupPreference": "Always", + "retentionInterval": "PT1H" + } + } + ] +} diff --git a/infrastructure/deployment_pg_azcn.json b/infrastructure/deployment_pg_azcn.json new file mode 100644 index 0000000..6b4fdb8 --- /dev/null +++ b/infrastructure/deployment_pg_azcn.json @@ -0,0 +1,646 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourcePrefix": { + "type": "string", + "metadata": { + "description": "provide a 2-13 character prefix for all resources. Be sure to lowercase." + } + }, + "AzureDatabaseForPostgreSQLName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-pgsql')]", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Name" + } + }, + "AzureDatabaseForPostgreSQLLocation": { + "type": "string", + "defaultValue": "chinanorth3", + "allowedValues": [ + "chinanorth3" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Location." + } + }, + "AzureDatabaseForPostgreSQLVersion": { + "type": "string", + "defaultValue": "16", + "allowedValues": [ + "15", + "16" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Version" + } + }, + "AzureDatabaseForPostgreSQLAdminUsername": { + "type": "string", + "defaultValue": "postgres", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Admin Username" + } + }, + "AzureDatabaseForPostgreSQLAdminPassword": { + "type": "securestring", + "defaultValue": "postgres", + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Admin Password" + } + }, + "AzureDatabaseForPostgreSQLSkuTier": { + "type": "string", + "defaultValue": "GeneralPurpose", + "allowedValues": [ + "GeneralPurpose", + "MemoryOptimized" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server SKU Tier" + } + }, + "AzureDatabaseForPostgreSQLSkuName": { + "type": "string", + "defaultValue": "Standard_E64ds_v4", + "allowedValues": [ + "Standard_D4ds_v4", + "Standard_D8ds_v4", + "Standard_D16ds_v4", + "Standard_D32ds_v4", + "Standard_D48ds_v4", + "Standard_D64ds_v4", + "Standard_E8ds_v4", + "Standard_E16ds_v4", + "Standard_E32ds_v4", + "Standard_E48ds_v4", + "Standard_E64ds_v4" + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server SKU Name" + } + }, + "AzureDatabaseForPostgreSQLStorageSize": { + "type": "int", + "defaultValue": 32768, + "allowedValues": [ + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768 + ], + "metadata": { + "description": "Azure Database for PostgreSQL Flexible Server Storage Size" + } + }, + "DocumentIntelligenceName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-doci')]", + "metadata": { + "description": "Document intelligence Name" + } + }, + "TranslatorName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-translator')]", + "metadata": { + "description": "Translator Name" + } + }, + "HostingPlanName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-plan')]", + "metadata": { + "description": "Name of App Service plan" + } + }, + "HostingPlanSku": { + "type": "string", + "defaultValue": "P3V2", + "allowedValues": [ + "B3", + "S2", + "S3", + "P2V2", + "P3V2" + ], + "metadata": { + "description": "The pricing tier for the App Service plan" + } + }, + "StorageAccountName": { + "type": "string", + "defaultValue": "[concat(toLower(parameters('ResourcePrefix')), 'str')]", + "metadata": { + "description": "Name of Storage Account" + } + }, + "WebsiteName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-site')]", + "metadata": { + "description": "Name of Web App" + } + }, + "FunctionName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-batchfunc')]", + "metadata": { + "description": "Name of Function App for Batch document processing" + } + }, + "ApplicationInsightsName": { + "type": "string", + "defaultValue": "[concat(parameters('ResourcePrefix'), '-appinsights')]", + "metadata": { + "description": "Name of Application Insights" + } + }, + "OpenAIName": { + "type": "string", + "metadata": { + "description": "Name of OpenAI Resource" + } + }, + "OpenAIKey": { + "type": "securestring", + "metadata": { + "description": "OpenAI API Key" + } + }, + "OpenAIEngine": { + "type": "string", + "metadata": { + "description": "OpenAI Engine" + } + }, + "OpenAIDeploymentType": { + "type": "string", + "allowedValues": [ + "Chat", + "Text" + ], + "metadata": { + "description": "OpenAI Deployment Type. Text for an Instructions based deployment (text-davinci-003). Chat for a Chat based deployment (gpt-35-turbo or gpt-4-32k or gpt-4)." + } + }, + "OpenAIEmbeddingsEngineDoc": { + "type": "string", + "defaultValue": "text-embedding-ada-002", + "metadata": { + "description": "OpenAI Embeddings Engine for Documents" + } + }, + "OpenAIEmbeddingsEngineQuery": { + "type": "string", + "defaultValue": "text-embedding-ada-002", + "metadata": { + "description": "OpenAI Embeddings Engine for Queries" + } + }, + "OpenAITemperature": { + "type": "string", + "defaultValue": "0.1", + "metadata": { + "description": "OpenAI Temperature" + } + }, + "OpenAIMaxTokens": { + "type": "string", + "defaultValue": "-1", + "metadata": { + "description": "OpenAI Max Tokens" + } + }, + "newGuid": { + "type": "string", + "defaultValue": "[newGuid()]" + } + }, + "variables": { + "VectorStoreType": "PGVector", + "BlobContainerName": "documents", + "QueueName": "doc-processing", + "ClientKey": "[concat(uniqueString(guid(resourceGroup().id, deployment().name)), parameters('newGuid'), 'Tg2%')]" + }, + "resources": [ + { + "type": "Microsoft.DBforPostgreSQL/flexibleServers", + "apiVersion": "2022-12-01", + "name": "[parameters('AzureDatabaseForPostgreSQLName')]", + "location": "[parameters('AzureDatabaseForPostgreSQLLocation')]", + "sku": { + "name": "[parameters('AzureDatabaseForPostgreSQLSkuName')]", + "tier": "[parameters('AzureDatabaseForPostgreSQLSkuTier')]" + }, + "properties": { + "administratorLogin": "[parameters('AzureDatabaseForPostgreSQLAdminUsername')]", + "administratorLoginPassword": "[parameters('AzureDatabaseForPostgreSQLAdminPassword')]", + "storage": { + "storageSizeGB": "[parameters('AzureDatabaseForPostgreSQLStorageSize')]" + }, + "createMode": "Default", + "version": "[parameters('AzureDatabaseForPostgreSQLVersion')]", + "backup": { + "backupRetentionDays": 35 + }, + "highAvailability": { + "mode": "Disabled" + }, + "publicNetworkAccess": "Enabled" + }, + "resources": [ + { + "type": "firewallRules", + "apiVersion": "2023-03-01-preview", + "name": "AllowAzureServices", + "dependsOn": [ + "[resourceId('Microsoft.DBforPostgreSQL/flexibleServers', parameters('AzureDatabaseForPostgreSQLName'))]" + ], + "properties": { + "startIpAddress": "0.0.0.0", + "endIpAddress": "0.0.0.0", + "startIpAddressRang": "AzureServices", + "endIpAddressRang": "AzureServices" + } + } + ] + }, + { + "type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations", + "apiVersion": "2022-12-01", + "name": "[concat(parameters('AzureDatabaseForPostgreSQLName'), '/azure.extensions')]", + "dependsOn": [ + "[resourceId('Microsoft.DBforPostgreSQL/flexibleServers', parameters('AzureDatabaseForPostgreSQLName'))]" + ], + "properties": { + "value": "vector", + "source": "user-override" + } + }, + { + "type": "Microsoft.CognitiveServices/accounts", + "apiVersion": "2022-12-01", + "name": "[parameters('DocumentIntelligenceName')]", + "location": "[resourceGroup().location]", + "sku": { + "name": "S0" + }, + "kind": "FormRecognizer", + "identity": { + "type": "None" + }, + "properties": { + "customSubDomainName": "[toLower(parameters('DocumentIntelligenceName'))]", + "networkAcls": { + "defaultAction": "Allow", + "virtualNetworkRules": [], + "ipRules": [] + }, + "publicNetworkAccess": "Enabled" + } + }, + { + "type": "Microsoft.CognitiveServices/accounts", + "apiVersion": "2022-12-01", + "name": "[parameters('TranslatorName')]", + "location": "[resourceGroup().location]", + "sku": { + "name": "S1" + }, + "kind": "TextTranslation", + "identity": { + "type": "None" + }, + "properties": { + "customSubDomainName": "[toLower(parameters('TranslatorName'))]", + "networkAcls": { + "defaultAction": "Allow", + "virtualNetworkRules": [], + "ipRules": [] + }, + "publicNetworkAccess": "Enabled" + } + }, + { + "apiVersion": "2020-06-01", + "name": "[parameters('HostingPlanName')]", + "type": "Microsoft.Web/serverfarms", + "location": "[resourceGroup().location]", + "sku": { + "name": "[parameters('HostingPlanSku')]" + }, + "properties": { + "name": "[parameters('HostingPlanName')]", + "reserved": true + }, + "kind": "linux" + }, + { + "apiVersion": "2020-06-01", + "name": "[parameters('WebsiteName')]", + "type": "Microsoft.Web/sites", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', parameters('HostingPlanName'))]" + ], + "properties": { + "serverFarmId": "[parameters('HostingPlanName')]", + "siteConfig": { + "linuxFxVersion": "DOCKER|cyberflying/aoai-web:latest" + } + } + }, + { + "name": "[parameters('StorageAccountName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-08-01", + "location": "[resourceGroup().location]", + "kind": "StorageV2", + "sku": { + "name": "Standard_GRS" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2021-08-01", + "name": "[concat(parameters('StorageAccountName'), '/default/', variables('BlobContainerName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "cors": { + "corsRules": [] + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices/queues", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default/doc-processing')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', parameters('StorageAccountName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "metadata": {} + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices/queues", + "apiVersion": "2022-09-01", + "name": "[concat(parameters('StorageAccountName'), '/default/doc-processing-poison')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', parameters('StorageAccountName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName'))]" + ], + "properties": { + "metadata": {} + } + }, + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[parameters('ApplicationInsightsName')]", + "location": "[resourceGroup().location]", + "tags": { + "[concat('hidden-link:', resourceId('Microsoft.Web/sites', parameters('ApplicationInsightsName')))]": "Resource" + }, + "properties": { + "Application_Type": "web" + }, + "kind": "web" + }, + { + "apiVersion": "2018-11-01", + "name": "[parameters('FunctionName')]", + "type": "Microsoft.Web/sites", + "kind": "functionapp,linux", + "location": "[resourceGroup().location]", + "tags": {}, + "dependsOn": [ + "[concat('Microsoft.Web/serverfarms/', parameters('HostingPlanName'))]", + "[concat('Microsoft.Storage/storageAccounts/', parameters('StorageAccountName'))]", + "[concat('Microsoft.Insights/components/', parameters('ApplicationInsightsName'))]" + ], + "properties": { + "name": "[parameters('FunctionName')]", + "siteConfig": { + "appSettings": [ + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "WEBSITES_ENABLE_APP_SERVICE_STORAGE", + "value": "false" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', parameters('ApplicationInsightsName')), '2015-05-01').InstrumentationKey]" + }, + { + "name": "AzureWebJobsStorage", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('StorageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.chinacloudapi.cn')]" + }, + { + "name": "OPENAI_ENGINE", + "value": "[parameters('OpenAIEngine')]" + }, + { + "name": "OPENAI_DEPLOYMENT_TYPE", + "value": "[parameters('OpenAIDeploymentType')]" + }, + { + "name": "OPENAI_EMBEDDINGS_ENGINE_DOC", + "value": "[parameters('OpenAIEmbeddingsEngineDoc')]" + }, + { + "name": "OPENAI_EMBEDDINGS_ENGINE_QUERY", + "value": "[parameters('OpenAIEmbeddingsEngineQuery')]" + }, + { + "name": "OPENAI_API_BASE", + "value": "[concat('https://', parameters('OpenAIName'), '.openai.azure.com/')]" + }, + { + "name": "OPENAI_API_KEY", + "value": "[parameters('OpenAIKey')]" + }, + { + "name": "OPENAI_TEMPERATURE", + "value": "[parameters('OpenAITemperature')]" + }, + { + "name": "OPENAI_MAX_TOKENS", + "value": "[parameters('OpenAIMaxTokens')]" + }, + { + "name": "BLOB_ACCOUNT_NAME", + "value": "[parameters('StorageAccountName')]" + }, + { + "name": "BLOB_ACCOUNT_KEY", + "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value]" + }, + { + "name": "BLOB_CONTAINER_NAME", + "value": "[variables('BlobContainerName')]" + }, + { + "name": "AZURE_CLOUD", + "value": "AzureChinaCloud" + }, + { + "name": "VECTOR_STORE_TYPE", + "value": "[variables('VectorStoreType')]" + }, + { + "name": "PGVECTOR_DRIVER", + "value": "psycopg2" + }, + { + "name": "PGVECTOR_HOST", + "value": "[concat(parameters('AzureDatabaseForPostgreSQLName'), '.postgres.database.chinacloudapi.cn')]" + }, + { + "name": "PGVECTOR_PORT", + "value": "5432" + }, + { + "name": "PGVECTOR_DATABASE", + "value": "postgres" + }, + { + "name": "PGVECTOR_USER", + "value": "[parameters('AzureDatabaseForPostgreSQLAdminUsername')]" + }, + { + "name": "PGVECTOR_PASSWORD", + "value": "[parameters('AzureDatabaseForPostgreSQLAdminPassword')]" + }, + { + "name": "FORM_RECOGNIZER_ENDPOINT", + "value": "[concat('https://',parameters('DocumentIntelligenceName'),'.cognitiveservices.azure.cn/')]" + }, + { + "name": "FORM_RECOGNIZER_KEY", + "value": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('DocumentIntelligenceName')), '2023-05-01').key1]" + }, + { + "name": "TRANSLATE_ENDPOINT", + "value": "https://api.translator.azure.cn/" + }, + { + "name": "TRANSLATE_KEY", + "value": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('TranslatorName')), '2023-05-01').key1]" + }, + { + "name": "TRANSLATE_REGION", + "value": "[resourceGroup().location]" + }, + { + "name": "QUEUE_NAME", + "value": "[variables('QueueName')]" + } + ], + "cors": { + "allowedOrigins": [ + "https://portal.azure.com" + ] + }, + "use32BitWorkerProcess": false, + "linuxFxVersion": "DOCKER|cyberflying/aoai-batch:latest", + "appCommandLine": "", + "alwaysOn": true + }, + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('HostingPlanName'))]", + "clientAffinityEnabled": false, + "virtualNetworkSubnetId": null, + "httpsOnly": true + } + }, + { + "type": "Microsoft.Web/sites/host/functionKeys", + "apiVersion": "2018-11-01", + "name": "[concat(parameters('FunctionName'), '/default/clientKey')]", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('FunctionName'))]", + "WaitFunctionDeploymentSection" + ], + "properties": { + "name": "ClientKey", + "value": "[variables('ClientKey')]" + } + }, + { + "type": "Microsoft.Web/sites/config", + "apiVersion": "2021-03-01", + "name": "[format('{0}/{1}', parameters('WebsiteName'), 'appsettings')]", + "kind": "string", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('WebsiteName'))]", + "[concat('Microsoft.Insights/components/', parameters('ApplicationInsightsName'))]" + ], + "properties": { + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.Insights/components', parameters('ApplicationInsightsName')), '2015-05-01').InstrumentationKey]", + "OPENAI_ENGINE": "[parameters('OpenAIEngine')]", + "OPENAI_DEPLOYMENT_TYPE": "[parameters('OpenAIDeploymentType')]", + "OPENAI_EMBEDDINGS_ENGINE_DOC": "[parameters('OpenAIEmbeddingsEngineDoc')]", + "OPENAI_EMBEDDINGS_ENGINE_QUERY": "[parameters('OpenAIEmbeddingsEngineQuery')]", + "AZURE_CLOUD": "AzureChinaCloud", + "VECTOR_STORE_TYPE": "[variables('VectorStoreType')]", + "PGVECTOR_DRIVER": "psycopg2", + "PGVECTOR_HOST": "[concat(parameters('AzureDatabaseForPostgreSQLName'), '.postgres.database.chinacloudapi.cn')]", + "PGVECTOR_PORT": "5432", + "PGVECTOR_DATABASE": "postgres", + "PGVECTOR_USER": "[parameters('AzureDatabaseForPostgreSQLAdminUsername')]", + "PGVECTOR_PASSWORD": "[parameters('AzureDatabaseForPostgreSQLAdminPassword')]", + "OPENAI_API_BASE": "[concat('https://', parameters('OpenAIName'),'.openai.azure.com/')]", + "OPENAI_API_KEY": "[parameters('OpenAIKey')]", + "OPENAI_TEMPERATURE": "[parameters('OpenAITemperature')]", + "OPENAI_MAX_TOKENS": "[parameters('OpenAIMaxTokens')]", + "BLOB_ACCOUNT_NAME": "[parameters('StorageAccountName')]", + "BLOB_ACCOUNT_KEY": "[listkeys(resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName')), '2015-05-01-preview').key1]", + "BLOB_CONTAINER_NAME": "[variables('BlobContainerName')]", + "FORM_RECOGNIZER_ENDPOINT": "[concat('https://',parameters('DocumentIntelligenceName'),'.cognitiveservices.azure.cn/')]", + "FORM_RECOGNIZER_KEY": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('DocumentIntelligenceName')), '2023-05-01').key1]", + "TRANSLATE_ENDPOINT": "https://api.translator.azure.cn/", + "TRANSLATE_KEY": "[listKeys(concat('Microsoft.CognitiveServices/accounts/', parameters('TranslatorName')), '2023-05-01').key1]", + "TRANSLATE_REGION": "[resourceGroup().location]", + "CONVERT_ADD_EMBEDDINGS_URL": "[concat('https://', parameters('FunctionName') , '.chinacloudsites.cn/api/BatchStartProcessing?code=', variables('ClientKey'))]" + } + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "kind": "AzurePowerShell", + "name": "WaitFunctionDeploymentSection", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', parameters('FunctionName'))]" + ], + "properties": { + "azPowerShellVersion": "3.0", + "scriptContent": "start-sleep -Seconds 100", + "cleanupPreference": "Always", + "retentionInterval": "PT1H" + } + } + ] +}