Skip to content

Commit

Permalink
[1/2] Rework test framework
Browse files Browse the repository at this point in the history
  • Loading branch information
ViliusSutkus89 committed Aug 20, 2024
1 parent a0c0d2e commit 36294d5
Show file tree
Hide file tree
Showing 4 changed files with 195 additions and 68 deletions.
115 changes: 48 additions & 67 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,21 +67,39 @@ jobs:
path: ~/.m2
if-no-files-found: error

instrumentedTests-aosp_atd-target:
# Only macos VMs are available with exposed CPU extensions to run hardware accelerated emulator
runs-on: macos-13
name: android-${{ matrix.api-level }} on ${{ matrix.arch }} (${{ matrix.api-type-target }})
needs: build
emulator:
runs-on: ubuntu-22.04
name: android-${{ matrix.emulator.api_level }} on ${{ matrix.emulator.arch }} (${{ matrix.emulator.api_type_target }})
strategy:
fail-fast: false
matrix:
api-level: [34, 33, 32, 31, 30]
arch: [x86_64]
api-type-target: [aosp_atd]
include:
- api-level: 30
arch: x86
api-type-target: aosp_atd
emulator:
# Emulator matrix generated by ci-scripts/emulator_matrix_generator.py
- { 'api_level': 35, 'api_type_target': 'google_apis', 'arch': 'x86_64' }
- { 'api_level': 34, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 33, 'api_type_target': 'aosp_atd', 'arch': 'x86_64' }
- { 'api_level': 32, 'api_type_target': 'aosp_atd', 'arch': 'x86_64' }
- { 'api_level': 31, 'api_type_target': 'aosp_atd', 'arch': 'x86_64' }
- { 'api_level': 30, 'api_type_target': 'aosp_atd', 'arch': 'x86_64' }
- { 'api_level': 30, 'api_type_target': 'aosp_atd', 'arch': 'x86' }
- { 'api_level': 29, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 29, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 28, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 28, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 27, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 27, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 26, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 26, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 25, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 25, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 24, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 24, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 23, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 23, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 22, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 22, 'api_type_target': 'default', 'arch': 'x86' }
- { 'api_level': 21, 'api_type_target': 'default', 'arch': 'x86_64' }
- { 'api_level': 21, 'api_type_target': 'default', 'arch': 'x86' }
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
Expand All @@ -90,68 +108,27 @@ jobs:
java-version: 17
- uses: android-actions/setup-android@v3

- uses: reactivecircus/android-emulator-runner@v2
with:
force-avd-creation: false
emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot-save
api-level: ${{ matrix.api-level }}
arch: ${{ matrix.arch }}
target: ${{ matrix.api-type-target }}
sdcard-path-or-size: 1G
disk-size: 8G
script: |
mkdir -p testResults/screenshots
adb logcat > testResults/logcat.txt &
adb shell screencap /data/local/tmp/beforeTests.png
adb pull /data/local/tmp/beforeTests.png testResults/screenshots/
./gradlew connectedCheck || touch sorry_but_tests_are_failing
adb pull /sdcard/Pictures/screenshots testResults/ || true
adb shell screencap /data/local/tmp/afterTests.png
adb pull /data/local/tmp/afterTests.png testResults/screenshots/
mv wvWare/build/reports/androidTests/connected testResults/
mv wvWare/build/outputs/androidTest-results testResults/
test ! -f sorry_but_tests_are_failing
- uses: actions/upload-artifact@v4
if: always()
- name: setup python 3.12
uses: actions/setup-python@v5
with:
name: testResults-${{ matrix.api-level }}-${{ matrix.arch }}-${{ matrix.api-type-target }}
path: testResults
if-no-files-found: error
python-version: 3.12
- run: pip install --upgrade adbPullAs selenium Pillow

instrumentedTests-default-target:
# Only macos VMs are available with exposed CPU extensions to run hardware accelerated emulator
runs-on: macos-13
name: android-${{ matrix.api-level }} on ${{ matrix.arch }} (${{ matrix.api-type-target }})
needs: build
strategy:
fail-fast: false
matrix:
api-level: [29, 28, 27, 26, 25, 24, 23, 22, 21]
arch: [x86_64, x86]
api-type-target: [default]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 17
- uses: android-actions/setup-android@v3
- name: Enable KVM group perms
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
- uses: reactivecircus/android-emulator-runner@v2
with:
force-avd-creation: false
emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot-save
api-level: ${{ matrix.api-level }}
arch: ${{ matrix.arch }}
target: ${{ matrix.api-type-target }}
sdcard-path-or-size: 1G
disk-size: 8G
disk-size: 7G
api-level: ${{ matrix.emulator.api_level }}
arch: ${{ matrix.emulator.arch }}
target: ${{ matrix.emulator.api_type_target }}
script: |
mkdir -p testResults/screenshots
adb logcat > testResults/logcat.txt &
Expand All @@ -160,6 +137,8 @@ jobs:
adb pull /data/local/tmp/beforeTests.png testResults/screenshots/
./gradlew connectedCheck || touch sorry_but_tests_are_failing
adbPullAs com.viliussutkus89.android.wvware.test /data/data/com.viliussutkus89.android.wvware.test/cache/wvWare testResults || true
adb pull /sdcard/Pictures/screenshots testResults/ || true
adb shell screencap /data/local/tmp/afterTests.png
Expand All @@ -170,9 +149,11 @@ jobs:
test ! -f sorry_but_tests_are_failing
- run: python ci-scripts/browser_tests.py --html-dir testResults/output-htmls --png-destination-dir testResults/pngs --reference-png-dir test/reference_pngs

- uses: actions/upload-artifact@v4
if: always()
with:
name: testResults-${{ matrix.api-level }}-${{ matrix.arch }}-${{ matrix.api-type-target }}
name: testResults-${{ matrix.emulator.api_level }}-${{ matrix.emulator.arch }}-${{ matrix.emulator.api_type_target }}
path: testResults
if-no-files-found: error
94 changes: 94 additions & 0 deletions ci-scripts/browser_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
# A lot of test logic taken from
# https://github.com/pdf2htmlEX/pdf2htmlEX/blob/v0.18.8.rc1/pdf2htmlEX/test/browser_tests.py

import argparse
import os
import sys
import time

from PIL import Image, ImageChops
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait


def main():
parser = argparse.ArgumentParser(description="Render test HTMLs into PNGs and compare against reference PNGs")
parser.add_argument("--html-dir", action="store", required=True,
help="Directory with converted HTMLs")
parser.add_argument("--png-destination-dir", action="store", required=True,
help="Directory where to save rendered PNGs")
parser.add_argument("--reference-png-dir", action="store", required=True,
help="Directory with reference PNGs")
args = parser.parse_args()
del parser

tests = ["sample", "Tom Taschauer"]

test_result = True

html_dir = os.path.abspath(args.html_dir)
reference_png_dir = os.path.abspath(args.reference_png_dir)
png_destination_dir = os.path.abspath(args.png_destination_dir)
if not os.path.exists(png_destination_dir):
os.mkdir(png_destination_dir, 0o755)

os.environ["MOZ_HEADLESS"] = "1"
for test in tests:
html_file = os.path.join(html_dir, f"{test}/{test}.html")
png_file = os.path.join(png_destination_dir, f"{test}.png")
reference_png_file = os.path.join(reference_png_dir, f"{test}.png")
diff_png_file = os.path.join(png_destination_dir, f"{test}-diff.png")

if not os.path.exists(html_file):
test_result = False
print(f" FAILURE: {test} html file not found!")
continue

browser = webdriver.Firefox()
browser.set_window_size(600, 800)
try:
browser.get('file://' + html_file)
WebDriverWait(browser, 5) \
.until(expected_conditions.presence_of_element_located((By.ID, 'page-container')))
finally:
time.sleep(1)
browser.get_full_page_screenshot_as_file(png_file)
browser.quit()

if not os.path.exists(reference_png_file):
print(f' IGNORED: {test} reference png file not found')
continue

out_img = Image.open(png_file).convert('RGB')
ref_img = Image.open(reference_png_file).convert('RGB')

diff_img = ImageChops.difference(ref_img, out_img)
diff_img.convert('RGB').save(diff_png_file)

diff_bbox = diff_img.getbbox()
if test == "test_fail":
if diff_bbox is None:
test_result = False
print(f" FAILURE: {test} should fail, but it did not. "
f"Test system is potentially broken.")
else:
print(f" SUCCESS: {test}")
elif diff_bbox is None:
print(f" SUCCESS: {test}")
else:
test_result = False
print(f" FAILURE: {test} diff bounding box: {diff_bbox} should be None")
diff_size = (diff_bbox[2] - diff_bbox[0]) * (diff_bbox[3] - diff_bbox[1])
img_size = ref_img.size[0] * ref_img.size[1]
print(f'PNG file {png_file} and {reference_png_file} differ by at'
f'most {diff_size} pixels, ({100.0*diff_size/img_size} of {img_size}'
f'pixels in total), difference: {diff_png_file}', file=sys.stderr)

sys.exit(0 if test_result else 1)


if __name__ == "__main__":
main()
53 changes: 53 additions & 0 deletions ci-scripts/emulator_matrix_generator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/usr/bin/env python3

import json
import os


def main():
# Rules for this generator:
#
# aosp_atd api_type_target available on emulator_api_level 30 and up,
# emulator_api_level 29 and below use default api_type_target
#
# x86 arch available on emulator_api_level 30 and lower.
#
# emulator_api_level 35 is available only on api_type_target google_apis.
# Don't use google_apis on previous emulator_api_levels
emulator_api_levels = (35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21)
arches = ("x86_64", "x86")
api_type_targets = ("aosp_atd", "default", "google_apis")

matrix = []
for emulator_api_level in emulator_api_levels:
for api_type_target in api_type_targets:
for arch in arches:
if emulator_api_level < 30 and api_type_target == "aosp_atd":
continue
if emulator_api_level >= 30 and api_type_target == "default":
continue
if emulator_api_level > 30 and arch == "x86":
continue
if emulator_api_level >= 35 and api_type_target != "google_apis":
continue
if emulator_api_level < 35 and api_type_target == "google_apis":
continue

matrix.append({
"api_level": emulator_api_level,
"api_type_target": api_type_target,
"arch": arch,
})

for i in matrix:
print('-', i)


gh_output = os.environ.get('GITHUB_OUTPUT')
if gh_output:
with open(gh_output, 'w') as out:
print("matrix=" + json.dumps(matrix), file=out)


if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@ public void cleanUp() {
if (null != convertedHtml) {
assertTrue("Converted HTML file not found!", convertedHtml.exists());
assertTrue("Converted HTML file empty!", convertedHtml.length() > 0);
convertedHtml.delete();
}
}

Expand Down

0 comments on commit 36294d5

Please sign in to comment.