Skip to content

Commit 9a5f4f3

Browse files
yuvipandaoliverroick
yuvipanda
and
oliverroick
committed
Add UI integration tests with playwright
- Add a new 'integration-tests' directory that does white box UI testing - Install dockerspawner in dev-requirements, as the playwright integration tests now need it - Move loading and about page tests to use playwright - Add some tests for the home page - Use a single instance of local-binder-local-hub for doing thes integration tests - Upload playwright traces (https://playwright.dev/python/docs/trace-viewer) on failure so we can debug things better Co-authored-by: Oliver Roick <[email protected]>
1 parent 386c11a commit 9a5f4f3

File tree

8 files changed

+312
-106
lines changed

8 files changed

+312
-106
lines changed

‎.github/workflows/playwright.yaml

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: Playwright Tests
2+
3+
on:
4+
pull_request:
5+
paths-ignore:
6+
- "**.md"
7+
- "**.rst"
8+
- "docs/**"
9+
- "examples/**"
10+
- ".github/workflows/**"
11+
- "!.github/workflows/playwright.yaml"
12+
push:
13+
paths-ignore:
14+
- "**.md"
15+
- "**.rst"
16+
- "docs/**"
17+
- "examples/**"
18+
- ".github/workflows/**"
19+
- "!.github/workflows/playwright.yaml"
20+
branches-ignore:
21+
- "dependabot/**"
22+
- "pre-commit-ci-update-config"
23+
- "update-*"
24+
workflow_dispatch:
25+
26+
jobs:
27+
tests:
28+
runs-on: ubuntu-22.04
29+
timeout-minutes: 10
30+
31+
permissions:
32+
contents: read
33+
env:
34+
GITHUB_ACCESS_TOKEN: "${{ secrets.github_token }}"
35+
36+
steps:
37+
- uses: actions/checkout@v4
38+
39+
- name: Setup OS level dependencies
40+
run: |
41+
sudo apt-get update
42+
sudo apt-get install --yes \
43+
build-essential \
44+
curl \
45+
libcurl4-openssl-dev \
46+
libssl-dev
47+
48+
- uses: actions/setup-node@v4
49+
id: setup-node
50+
with:
51+
node-version: "22"
52+
53+
- name: Cache npm
54+
uses: actions/cache@v4
55+
with:
56+
path: ~/.npm
57+
key: node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('**/package.json') }}-${{ github.job }}
58+
59+
- name: Run webpack to build static assets
60+
run: |
61+
npm install
62+
npm run webpack
63+
64+
- uses: actions/setup-python@v5
65+
id: setup-python
66+
with:
67+
python-version: "3.12"
68+
69+
- name: Cache pip
70+
uses: actions/cache@v4
71+
with:
72+
path: ~/.cache/pip
73+
key: python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }}-${{ github.job }}
74+
75+
- name: Setup test dependencies
76+
run: |
77+
npm i -g configurable-http-proxy
78+
79+
pip install --no-binary pycurl -r dev-requirements.txt
80+
pip install -e .
81+
82+
- name: Install playwright browser
83+
run: |
84+
playwright install firefox
85+
86+
- name: Run playwright tests
87+
run: |
88+
py.test --cov=binderhub -s integration-tests/
89+
90+
- uses: actions/upload-artifact@v4
91+
if: always()
92+
with:
93+
name: playwright-traces
94+
path: test-results/
95+
96+
# Upload test coverage info to codecov
97+
- uses: codecov/codecov-action@v5

‎binderhub/tests/conftest.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,12 @@
4242

4343

4444
def pytest_configure(config):
45-
"""This function has meaning to pytest, for more information, see:
46-
https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure
45+
"""
46+
Configure plugins and custom markers
47+
48+
This function is called by pytest after command line arguments have
49+
been parsed. See https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_configure
50+
for more information.
4751
"""
4852
# register our custom markers
4953
config.addinivalue_line(

‎binderhub/tests/test_main.py

Lines changed: 0 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -60,29 +60,6 @@ def _resolve_url(page_url, url):
6060
return f"{parsed.scheme}://{parsed.netloc}{path}{url}"
6161

6262

63-
@pytest.mark.remote
64-
async def test_main_page(app):
65-
"""Check the main page and any links on it"""
66-
r = await async_requests.get(app.url)
67-
assert r.status_code == 200
68-
soup = BeautifulSoup(r.text, "html5lib")
69-
70-
# check src links (style, images)
71-
for el in soup.find_all(src=True):
72-
url = _resolve_url(app.url, el["src"])
73-
r = await async_requests.get(url)
74-
assert r.status_code == 200, f"{r.status_code} {url}"
75-
76-
# check hrefs
77-
for el in soup.find_all(href=True):
78-
href = el["href"]
79-
if href.startswith("#"):
80-
continue
81-
url = _resolve_url(app.url, href)
82-
r = await async_requests.get(url)
83-
assert r.status_code == 200, f"{r.status_code} {url}"
84-
85-
8663
@pytest.mark.remote
8764
@pytest.mark.helm
8865
async def test_custom_template(app):
@@ -92,15 +69,6 @@ async def test_custom_template(app):
9269
assert "test-template" in r.text
9370

9471

95-
@pytest.mark.remote
96-
async def test_about_handler(app):
97-
# Check that the about page loads
98-
r = await async_requests.get(app.url + "/about")
99-
assert r.status_code == 200
100-
assert "This website is powered by" in r.text
101-
assert binder_version.split("+")[0] in r.text
102-
103-
10472
@pytest.mark.remote
10573
async def test_versions_handler(app):
10674
# Check that the about page loads
@@ -121,65 +89,6 @@ async def test_versions_handler(app):
12189
assert data["binderhub"].split("+")[0] == binder_version.split("+")[0]
12290

12391

124-
@pytest.mark.parametrize(
125-
"provider_prefix,repo,ref,path,path_type,status_code",
126-
[
127-
("gh", "binderhub-ci-repos/requirements", "master", "", "", 200),
128-
("gh", "binderhub-ci-repos%2Frequirements", "master", "", "", 400),
129-
("gh", "binderhub-ci-repos/requirements", "master/", "", "", 200),
130-
(
131-
"gh",
132-
"binderhub-ci-repos/requirements",
133-
"20c4fe55a9b2c5011d228545e821b1c7b1723652",
134-
"index.ipynb",
135-
"file",
136-
200,
137-
),
138-
(
139-
"gh",
140-
"binderhub-ci-repos/requirements",
141-
"20c4fe55a9b2c5011d228545e821b1c7b1723652",
142-
"%2Fnotebooks%2Findex.ipynb",
143-
"url",
144-
200,
145-
),
146-
("gh", "binderhub-ci-repos/requirements", "master", "has%20space", "file", 200),
147-
(
148-
"gh",
149-
"binderhub-ci-repos/requirements",
150-
"master/",
151-
"%2Fhas%20space%2F",
152-
"file",
153-
200,
154-
),
155-
(
156-
"gh",
157-
"binderhub-ci-repos/requirements",
158-
"master",
159-
"%2Fhas%20space%2F%C3%BCnicode.ipynb",
160-
"file",
161-
200,
162-
),
163-
],
164-
)
165-
async def test_loading_page(
166-
app, provider_prefix, repo, ref, path, path_type, status_code
167-
):
168-
# repo = f'{org}/{repo_name}'
169-
spec = f"{repo}/{ref}"
170-
provider_spec = f"{provider_prefix}/{spec}"
171-
query = f"{path_type}path={path}" if path else ""
172-
uri = f"/v2/{provider_spec}?{query}"
173-
r = await async_requests.get(app.url + uri)
174-
assert r.status_code == status_code, f"{r.status_code} {uri}"
175-
if status_code == 200:
176-
soup = BeautifulSoup(r.text, "html5lib")
177-
assert soup.find(id="log-container")
178-
nbviewer_url = soup.find(id="nbviewer-preview").find("iframe").attrs["src"]
179-
r = await async_requests.get(nbviewer_url)
180-
assert r.status_code == 200, f"{r.status_code} {nbviewer_url}"
181-
182-
18392
@pytest.mark.parametrize(
18493
"origin,host,expected_origin",
18594
[

‎binderhub/tests/utils.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import asyncio
44
import io
5+
import socket
56
from concurrent.futures import ThreadPoolExecutor
67
from urllib.parse import urlparse
78

@@ -149,3 +150,12 @@ def _next():
149150

150151
# async_requests.get = requests.get returning a Future, etc.
151152
async_requests = _AsyncRequests()
153+
154+
155+
def random_port() -> int:
156+
"""Get a single random port."""
157+
sock = socket.socket()
158+
sock.bind(("", 0))
159+
port = sock.getsockname()[1]
160+
sock.close()
161+
return port

‎dev-requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@ beautifulsoup4[html5lib]
22
build
33
chartpress>=2.1
44
click
5+
dockerspawner
56
jsonschema
67
jupyter-repo2docker>=2021.08.0
78
jupyter_packaging>=0.10.4,<2
89
jupyterhub
10+
nest-asyncio
911
pytest
1012
pytest-asyncio
1113
pytest-cov
1214
pytest-timeout
15+
pytest_playwright
1316
requests
1417
ruamel.yaml>=0.17.30

‎integration-tests/conftest.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import nest_asyncio
2+
3+
4+
def pytest_configure(config):
5+
# Required for playwright to be run from within pytest
6+
nest_asyncio.apply()

0 commit comments

Comments
 (0)