From 889af0478442e6fd7a9cf2780dafcf9461186399 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Mon, 3 Feb 2025 16:32:26 +0100 Subject: [PATCH] Remove blocked_pages check (see mozilla-services/cloudops-infra#5052) --- checks/remotesettings/blocked_pages.py | 72 -------------- .../remotesettings/test_blocked_pages.py | 95 ------------------- 2 files changed, 167 deletions(-) delete mode 100644 checks/remotesettings/blocked_pages.py delete mode 100644 tests/checks/remotesettings/test_blocked_pages.py diff --git a/checks/remotesettings/blocked_pages.py b/checks/remotesettings/blocked_pages.py deleted file mode 100644 index 075a6ec0..00000000 --- a/checks/remotesettings/blocked_pages.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -The HTML content of the page that lists blocked addons and plugins should -match the source of truth. - -The list of missing or extras entries is returned, along with the source -timestamps. -""" - -import logging -import re - -import aiohttp -from bs4 import BeautifulSoup - -from telescope.typings import CheckResult -from telescope.utils import fetch_head, fetch_text, run_parallel - -from .utils import KintoClient - - -EXPOSED_PARAMETERS = ["remotesettings_server", "blocked_pages"] - -logger = logging.getLogger(__name__) - - -async def test_url(url): - try: - status, _ = await fetch_head(url) - return status == 200 - except aiohttp.ClientError: - return False - - -async def run(remotesettings_server: str, blocked_pages: str) -> CheckResult: - # Read blocked page index to obtain the links. - blocked_index = await fetch_text(blocked_pages) - soup = BeautifulSoup(blocked_index, features="html.parser") - urls = [] - for link in soup.find_all("a", href=re.compile(".html$")): - urls.append(link["href"]) - - # Make sure no link is broken. - futures = [test_url(f"{blocked_pages}/{url}") for url in urls] - results = await run_parallel(*futures) - urls_success = zip(urls, results) - missing = [url for url, success in urls_success if not success] - - # Compare list of blocked ids with the source of truth. - client = KintoClient(server_url=remotesettings_server, bucket="blocklists") - addons_records = await client.get_records(collection="addons") - plugins_records = await client.get_records(collection="plugins") - records_ids = [r.get("blockID", r["id"]) for r in plugins_records + addons_records] - blocked_ids = [url.rsplit(".", 1)[0] for url in urls] - extras_ids = set(blocked_ids) - set(records_ids) - missing_ids = set(records_ids) - set(blocked_ids) - - addons_timestamp = int(await client.get_records_timestamp(collection="addons")) - plugins_timestamp = int(await client.get_records_timestamp(collection="plugins")) - certificates_timestamp = int( - await client.get_records_timestamp(collection="certificates") - ) - - success = len(missing) == 0 and len(missing_ids) == 0 and len(extras_ids) == 0 - data = { - "addons-timestamp": addons_timestamp, - "plugins-timestamp": plugins_timestamp, - "certificates-timestamp": certificates_timestamp, - "broken-links": missing, - "missing": list(missing_ids), - "extras": list(extras_ids), - } - return success, data diff --git a/tests/checks/remotesettings/test_blocked_pages.py b/tests/checks/remotesettings/test_blocked_pages.py deleted file mode 100644 index 43a71d82..00000000 --- a/tests/checks/remotesettings/test_blocked_pages.py +++ /dev/null @@ -1,95 +0,0 @@ -from checks.remotesettings.blocked_pages import run - - -COLLECTION_URL = "/buckets/{}/collections/{}" -RECORDS_URL = COLLECTION_URL + "/records" - - -def mock_kinto_responses(mock_responses, server_url): - mock_responses.get( - server_url + RECORDS_URL.format("blocklists", "plugins"), - payload={"data": [{"id": "1-2-3", "blockID": "abc"}, {"id": "4-5-6"}]}, - headers={"ETag": '"157556192042"'}, - ) - mock_responses.get( - server_url + RECORDS_URL.format("blocklists", "addons"), - payload={ - "data": [{"id": "def", "blockID": "7-8-9", "last_modified": 1568816392824}] - }, - headers={"ETag": '"1568816392824"'}, - ) - mock_responses.head( - server_url + RECORDS_URL.format("blocklists", "certificates"), - headers={"ETag": '"1181628381652"'}, - ) - - -async def test_positive(mock_aioresponses, mock_responses): - server_url = "http://fake.local/v1" - blocked_url = "http://blocked.cdn" - - mock_kinto_responses(mock_responses, server_url) - - page_content = """ - - - - - """ - mock_aioresponses.get(blocked_url + "/", body=page_content) - mock_aioresponses.head(blocked_url + "/abc.html") - mock_aioresponses.head(blocked_url + "/4-5-6.html") - mock_aioresponses.head(blocked_url + "/7-8-9.html") - - status, data = await run( - remotesettings_server=server_url, blocked_pages=blocked_url - ) - - assert status is True - assert data == { - "addons-timestamp": 1568816392824, - "plugins-timestamp": 157556192042, - "certificates-timestamp": 1181628381652, - "broken-links": [], - "missing": [], - "extras": [], - } - - -async def test_negative(mock_aioresponses, mock_responses): - server_url = "http://fake.local/v1" - blocked_url = "http://blocked.cdn" - - mock_kinto_responses(mock_responses, server_url) - - page_content = """ - - - - - """ - mock_aioresponses.get(blocked_url + "/", body=page_content) - mock_aioresponses.head(blocked_url + "/4-5-6.html") - mock_aioresponses.head(blocked_url + "/extra.html") - - status, data = await run( - remotesettings_server=server_url, blocked_pages=blocked_url - ) - - assert status is False - assert data == { - "addons-timestamp": 1568816392824, - "plugins-timestamp": 157556192042, - "certificates-timestamp": 1181628381652, - "broken-links": ["7-8-9.html"], - "missing": ["abc"], - "extras": ["extra"], - }