Vizro ai response time detection

.github/workflows/test-integration-vizro-ai.yml

@@ -7,6 +7,9 @@ defaults:
 on:
  push:
  branches: [main]
+ pull_request:
+ branches:
+ - main
  schedule:
  - cron: "30 10 * * 1" # run every Monday at 10:30 UTC
 
@@ -26,15 +29,15 @@ jobs:
  include:
  - python-version: "3.9"
  hatch-env: all.py3.9
- - python-version: "3.10"
- hatch-env: all.py3.10
- - python-version: "3.11"
- hatch-env: all.py3.11
- - python-version: "3.12"
- hatch-env: all.py3.12
- - python-version: "3.11"
- hatch-env: lower-bounds
- label: lower bounds
+ #  - python-version: "3.10"
+ #  hatch-env: all.py3.10
+ #  - python-version: "3.11"
+ #  hatch-env: all.py3.11
+ #  - python-version: "3.12"
+ #  hatch-env: all.py3.12
+ #  - python-version: "3.11"
+ #  hatch-env: lower-bounds
+ #  label: lower bounds
 
  steps:
  - uses: actions/checkout@v4
@@ -54,6 +57,11 @@ jobs:
 
  - name: Run vizro-ai integration tests with pypi vizro
  run: |
+ touch report_test_chart_$(date '+%Y-%m-%d').csv
+ touch report_test_chart_with_explanation_$(date '+%Y-%m-%d').csv
+ str="GPT Model, Resp. time"
+ echo $str >> report_test_chart_$(date '+%Y-%m-%d').csv
+ echo report_test_chart_with_explanation_$(date '+%Y-%m-%d').csv
  export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
  export OPENAI_API_BASE=${{ secrets.OPENAI_API_BASE }}
  hatch run ${{ matrix.hatch-env }}:test-integration
@@ -67,3 +75,11 @@ jobs:
  cd ../vizro-ai
  hatch run ${{ matrix.hatch-env }}:pip install ../vizro-core/dist/vizro*.tar.gz
  hatch run ${{ matrix.hatch-env }}:test-integration
+
+ - name: Report artifacts
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: Report
+ path: |
+ /home/runner/work/vizro/vizro/vizro-ai/report*.csv

vizro-ai/changelog.d/20240322_145222_alexey_snigir_vizro_ai_resp_time.md

@@ -0,0 +1,48 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+-->
+
+<!--
+### Highlights ✨
+
+- A bullet item for the Highlights ✨ category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Added
+
+- A bullet item for the Added category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Changed
+
+- A bullet item for the Changed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->

vizro-ai/hatch.toml

@@ -34,7 +34,7 @@ prep-release = [
 pypath = "hatch run python -c 'import sys; print(sys.executable)'"
 secrets = "pre-commit run gitleaks --all-files"
 test = "pytest tests {args}"
-test-integration = "pytest -v tests/integration {args}"
+test-integration = "pytest -vs tests/integration {args}"
 test-unit = "pytest tests/unit {args}"
 test-unit-coverage = [
  "coverage run -m pytest tests/unit {args}",

vizro-ai/tests/integration/test_example.py

@@ -1,25 +1,55 @@
+import csv
+import time
+from datetime import datetime
+
 import plotly.express as px
+import pytest
 from hamcrest import all_of, any_of, assert_that, contains_string, equal_to
 from vizro_ai import VizroAI
 
 vizro_ai = VizroAI()
 df = px.data.gapminder()
+today = datetime.today()
+formatted_date = today.strftime("%Y-%m-%d")
 
 
-def test_chart():
+@pytest.mark.parametrize(
+ "model_name",
+ ["gpt-3.5-turbo", "gpt-4-0613"],
+ ids=["gpt-3.5", "gpt-4.0"],
+)
+def test_chart(model_name):
  vizro_ai._return_all_text = True
+ vizro_ai.model_name = model_name
+ before = time.time()
  resp = vizro_ai.plot(df, "describe the composition of scatter chart with gdp in continent")
+ after = time.time()
+ resp_time = after - before
+ print("RESPONSE TIME:", after - before) # noqa: T201
  assert_that(
  resp["code_string"],
  all_of(contains_string("px.scatter"), contains_string("x='continent'"), contains_string("y='gdpPercap'")),
  )
  assert_that(resp["business_insights"], equal_to(None))
  assert_that(resp["code_explanation"], equal_to(None))
+ with open(f"report_test_chart_{formatted_date}.csv", "a", newline="") as csvfile:
+ writer = csv.writer(csvfile, delimiter=",", quotechar="|", quoting=csv.QUOTE_MINIMAL)
+ writer.writerow([f"{model_name}", f"{resp_time}"])
 
 
-def test_chart_with_explanation():
+@pytest.mark.parametrize(
+ "model_name",
+ ["gpt-3.5-turbo", "gpt-4-0613"],
+ ids=["gpt-3.5", "gpt-4.0"],
+)
+def test_chart_with_explanation(model_name):
  vizro_ai._return_all_text = True
+ vizro_ai.model_name = model_name
+ before = time.time()
  resp = vizro_ai.plot(df, "describe the composition of gdp in US", explain=True)
+ after = time.time()
+ resp_time = after - before
+ print("RESPONSE TIME (explanation):", after - before) # noqa: T201
  assert_that(
  resp["code_string"],
  all_of(contains_string("px.bar"), contains_string("x='year'")),
@@ -40,3 +70,6 @@ def test_chart_with_explanation():
  resp["code_explanation"],
  all_of(contains_string("https://vizro.readthedocs.io/en/stable/pages/user_guides/custom_charts/")),
  )
+ with open(f"report_test_chart_with_explanation_{formatted_date}.csv", "a", newline="") as csvfile:
+ writer = csv.writer(csvfile, delimiter=",", quotechar="|", quoting=csv.QUOTE_MINIMAL)
+ writer.writerow([f"{model_name}", f"{resp_time}"])