Skip to content

Commit 762662a

Browse files
leplatremleplatrem
authored
Leverage newest features of kinto-http.py (#1515)
* Use collection_diff * Use get_changeset() * Lint
1 parent 4ddbcfb commit 762662a

14 files changed

+122
-131
lines changed

‎.secrets.baseline‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@
9090
{
9191
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
9292
},
93+
{
94+
"path": "detect_secrets.filters.common.is_baseline_file",
95+
"filename": ".secrets.baseline"
96+
},
9397
{
9498
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
9599
"min_level": 2
@@ -138,14 +142,14 @@
138142
"filename": "tests/checks/remotesettings/test_attachments_integrity.py",
139143
"hashed_secret": "263e1869fb57deeb75844cf85d55f0d03a6019fc",
140144
"is_verified": false,
141-
"line_number": 31
145+
"line_number": 32
142146
},
143147
{
144148
"type": "Hex High Entropy String",
145149
"filename": "tests/checks/remotesettings/test_attachments_integrity.py",
146150
"hashed_secret": "fdf34abe05071170ce2ec082a18aa762ec454ae3",
147151
"is_verified": false,
148-
"line_number": 39
152+
"line_number": 40
149153
}
150154
],
151155
"tests/checks/remotesettings/test_public_suffix_list.py": [
@@ -158,5 +162,5 @@
158162
}
159163
]
160164
},
161-
"generated_at": "2024-10-18T09:21:04Z"
165+
"generated_at": "2024-12-03T11:11:17Z"
162166
}

‎checks/remotesettings/backported_records.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@
99
from typing import Any, Dict
1010
from urllib.parse import parse_qs
1111

12+
from kinto_http.utils import collection_diff
13+
1214
from telescope.typings import CheckResult
1315

14-
from .utils import KintoClient, compare_collections, human_diff
16+
from .utils import KintoClient, human_diff
1517

1618

1719
EXPOSED_PARAMETERS = ["server", "max_lag_seconds"]
@@ -40,8 +42,8 @@ async def run(
4042
dest_records = await client.get_records(
4143
bucket=dest_bid, collection=dest_cid, _expected="Foo"
4244
)
43-
diff = compare_collections(source_records, dest_records)
44-
if diff:
45+
to_create, to_update, to_delete = collection_diff(source_records, dest_records)
46+
if to_create or to_update or to_delete:
4547
source_timestamp = await client.get_records_timestamp(
4648
bucket=source_bid, collection=source_cid
4749
)
@@ -50,8 +52,7 @@ async def run(
5052
)
5153
diff_millisecond = abs(int(source_timestamp) - int(dest_timestamp))
5254
if (diff_millisecond / 1000) > max_lag_seconds:
53-
missing, differ, extras = diff
54-
details = human_diff(source, dest, missing, differ, extras)
55+
details = human_diff(source, dest, to_create, to_update, to_delete)
5556
errors.append(details)
5657

5758
return len(errors) == 0, errors

‎checks/remotesettings/collections_consistency.py‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77

88
import logging
99

10+
from kinto_http.utils import collection_diff
11+
1012
from telescope.typings import CheckResult
1113
from telescope.utils import run_parallel
1214

13-
from .utils import KintoClient, compare_collections, fetch_signed_resources, human_diff
15+
from .utils import KintoClient, fetch_signed_resources, human_diff
1416

1517

1618
EXPOSED_PARAMETERS = ["server"]
@@ -46,9 +48,13 @@ async def has_inconsistencies(server_url, auth, resource):
4648

4749
source_records = await client.get_records(**source)
4850
preview_records = await client.get_records(**resource["preview"])
49-
diff = compare_collections(source_records, preview_records)
50-
if diff:
51-
return message + human_diff("source", "preview", *diff)
51+
to_create, to_update, to_delete = collection_diff(
52+
source_records, preview_records
53+
)
54+
if to_create or to_update or to_delete:
55+
return message + human_diff(
56+
"source", "preview", to_create, to_update, to_delete
57+
)
5258

5359
# And if status is ``signed``, then records in the source and preview should
5460
# all be the same as those in the destination.
@@ -59,18 +65,30 @@ async def has_inconsistencies(server_url, auth, resource):
5965
# If preview is enabled, then compare source/preview and preview/dest
6066
preview_records = await client.get_records(**resource["preview"])
6167

62-
diff_preview = compare_collections(preview_records, dest_records)
63-
if diff_preview:
64-
return message + human_diff("preview", "destination", *diff_preview)
68+
to_create, to_update, to_delete = collection_diff(
69+
preview_records, dest_records
70+
)
71+
if to_create or to_update or to_delete:
72+
return message + human_diff(
73+
"preview", "destination", to_create, to_update, to_delete
74+
)
6575

66-
diff_source = compare_collections(source_records, preview_records)
67-
if diff_source:
68-
return message + human_diff("source", "preview", *diff_source)
76+
to_create, to_update, to_delete = collection_diff(
77+
source_records, preview_records
78+
)
79+
if to_create or to_update or to_delete:
80+
return message + human_diff(
81+
"source", "preview", to_create, to_update, to_delete
82+
)
6983
else:
7084
# Otherwise, just compare source/dest
71-
diff_source = compare_collections(source_records, dest_records)
72-
if diff_source:
73-
return message + human_diff("source", "destination", *diff_source)
85+
to_create, to_update, to_delete = collection_diff(
86+
source_records, dest_records
87+
)
88+
if to_create or to_update or to_delete:
89+
return message + human_diff(
90+
"source", "destination", to_create, to_update, to_delete
91+
)
7492

7593
elif status == "work-in-progress":
7694
# And if status is ``work-in-progress``, we can't really check anything.

‎checks/remotesettings/utils.py‎

Lines changed: 22 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
1-
import asyncio
21
import copy
3-
import random
42
import re
5-
from typing import Dict, List, Optional, Tuple
3+
from typing import Any, Dict, List
64

75
import backoff
86
import kinto_http
@@ -49,23 +47,15 @@ async def get_records(self, *args, **kwargs) -> List[Dict]:
4947
return await self._client.get_records(*args, **kwargs)
5048

5149
@retry_timeout
52-
async def get_monitor_changes(self, bust_cache=False, **kwargs) -> List[Dict]:
53-
if bust_cache:
54-
if "_expected" in kwargs:
55-
raise ValueError("Pick one of `bust_cache` and `_expected` parameters")
56-
random_cache_bust = random.randint(999999000000, 999999999999)
57-
kwargs["_expected"] = random_cache_bust
58-
return await self.get_records(bucket="monitor", collection="changes", **kwargs)
50+
async def get_monitor_changes(self, **kwargs) -> List[Dict]:
51+
resp = await self.get_changeset(
52+
bucket="monitor", collection="changes", **kwargs
53+
)
54+
return resp["changes"]
5955

6056
@retry_timeout
61-
async def get_changeset(self, bucket, collection, **kwargs) -> List[Dict]:
62-
endpoint = f"/buckets/{bucket}/collections/{collection}/changeset"
63-
kwargs.setdefault("_expected", random.randint(999999000000, 999999999999))
64-
loop = asyncio.get_event_loop()
65-
body, _ = await loop.run_in_executor(
66-
None, lambda: self._client.session.request("get", endpoint, params=kwargs)
67-
)
68-
return body
57+
async def get_changeset(self, *args, **kwargs) -> Dict[str, Any]:
58+
return await self._client.get_changeset(*args, **kwargs)
6959

7060
@retry_timeout
7161
async def get_record(self, *args, **kwargs) -> Dict:
@@ -110,9 +100,7 @@ async def fetch_signed_resources(server_url: str, auth: str) -> List[Dict[str, D
110100
preview_buckets.add(resource["preview"]["bucket"])
111101

112102
resources = []
113-
monitored = await client.get_records(
114-
bucket="monitor", collection="changes", _sort="bucket,collection"
115-
)
103+
monitored = await client.get_monitor_changes(_sort="bucket,collection")
116104
for entry in monitored:
117105
bid = entry["bucket"]
118106
cid = entry["collection"]
@@ -138,60 +126,35 @@ async def fetch_signed_resources(server_url: str, auth: str) -> List[Dict[str, D
138126
return resources
139127

140128

141-
def records_equal(a, b):
142-
"""Compare records, ignoring timestamps."""
143-
ignored_fields = ("last_modified", "schema")
144-
ra = {k: v for k, v in a.items() if k not in ignored_fields}
145-
rb = {k: v for k, v in b.items() if k not in ignored_fields}
146-
return ra == rb
147-
148-
149-
def compare_collections(
150-
a: List[Dict], b: List[Dict]
151-
) -> Optional[Tuple[List[str], List[str], List[str]]]:
152-
"""Compare two lists of records. Returns empty list if equal."""
153-
b_by_id = {r["id"]: r for r in b}
154-
missing = []
155-
differ = []
156-
for ra in a:
157-
rb = b_by_id.pop(ra["id"], None)
158-
if rb is None:
159-
missing.append(ra["id"])
160-
elif not records_equal(ra, rb):
161-
differ.append(ra["id"])
162-
extras = list(b_by_id.keys())
163-
164-
if missing or differ or extras:
165-
return (missing, differ, extras)
166-
167-
return None
168-
169-
170129
def human_diff(
171130
left: str,
172131
right: str,
173-
missing: List[str],
174-
differ: List[str],
175-
extras: List[str],
132+
missing: List[dict],
133+
differ: List[tuple[dict, dict]],
134+
extras: List[dict],
176135
show_ids: int = 5,
177136
) -> str:
137+
missing_ids = [r["id"] for r in missing]
138+
differ_ids = [r["id"] for _, r in differ]
139+
extras_ids = [r["id"] for r in extras]
140+
178141
def ellipse(line):
179142
return ", ".join(repr(r) for r in line[:show_ids]) + (
180143
"..." if len(line) > show_ids else ""
181144
)
182145

183146
details = []
184-
if missing:
147+
if missing_ids:
185148
details.append(
186-
f"{len(missing)} record{'s' if len(missing) > 1 else ''} present in {left} but missing in {right} ({ellipse(missing)})"
149+
f"{len(missing_ids)} record{'s' if len(missing_ids) > 1 else ''} present in {left} but missing in {right} ({ellipse(missing_ids)})"
187150
)
188-
if differ:
151+
if differ_ids:
189152
details.append(
190-
f"{len(differ)} record{'s' if len(differ) > 1 else ''} differ between {left} and {right} ({ellipse(differ)})"
153+
f"{len(differ_ids)} record{'s' if len(differ_ids) > 1 else ''} differ between {left} and {right} ({ellipse(differ_ids)})"
191154
)
192-
if extras:
155+
if extras_ids:
193156
details.append(
194-
f"{len(extras)} record{'s' if len(extras) > 1 else ''} present in {right} but missing in {left} ({ellipse(extras)})"
157+
f"{len(extras_ids)} record{'s' if len(extras_ids) > 1 else ''} present in {right} but missing in {left} ({ellipse(extras_ids)})"
195158
)
196159
return ", ".join(details)
197160

‎checks/remotesettings/work_in_progress.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@
99
import sys
1010
from datetime import datetime
1111

12+
from kinto_http.utils import collection_diff
13+
1214
from telescope.typings import CheckResult
1315
from telescope.utils import run_parallel, utcnow
1416

15-
from .utils import KintoClient, compare_collections, fetch_signed_resources
17+
from .utils import KintoClient, fetch_signed_resources
1618

1719

1820
logger = logging.getLogger(__name__)
@@ -62,7 +64,10 @@ async def run(server: str, auth: str, max_age: int) -> CheckResult:
6264
# These collections are worth introspecting.
6365
source_records = await client.get_records(**resource["source"])
6466
destination_records = await client.get_records(**resource["destination"])
65-
if not compare_collections(source_records, destination_records):
67+
to_create, to_update, to_delete = collection_diff(
68+
source_records, destination_records
69+
)
70+
if not (to_create or to_update or to_delete):
6671
continue
6772

6873
# Fetch list of editors, if necessary to contact them.

‎tests/checks/remotesettings/test_attachments_availability.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from checks.remotesettings.attachments_availability import run
22

33

4+
CHANGESET_URL = "/buckets/{}/collections/{}/changeset"
45
RECORDS_URL = "/buckets/{}/collections/{}/records"
56

67

@@ -10,11 +11,11 @@ async def test_positive(mock_responses, mock_aioresponses):
1011
server_url + "/",
1112
payload={"capabilities": {"attachments": {"base_url": "http://cdn/"}}},
1213
)
13-
changes_url = server_url + RECORDS_URL.format("monitor", "changes")
14+
changes_url = server_url + CHANGESET_URL.format("monitor", "changes")
1415
mock_responses.get(
1516
changes_url,
1617
payload={
17-
"data": [
18+
"changes": [
1819
{"id": "abc", "bucket": "bid", "collection": "cid", "last_modified": 42}
1920
]
2021
},
@@ -45,11 +46,11 @@ async def test_negative(mock_responses, mock_aioresponses):
4546
server_url + "/",
4647
payload={"capabilities": {"attachments": {"base_url": "http://cdn/"}}},
4748
)
48-
changes_url = server_url + RECORDS_URL.format("monitor", "changes")
49+
changes_url = server_url + CHANGESET_URL.format("monitor", "changes")
4950
mock_responses.get(
5051
changes_url,
5152
payload={
52-
"data": [
53+
"changes": [
5354
{"id": "abc", "bucket": "bid", "collection": "cid", "last_modified": 42}
5455
]
5556
},

‎tests/checks/remotesettings/test_attachments_bundles.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,11 @@ async def test_negative(mock_responses, mock_aioresponses):
4141
may8_http = "Mon, 08 May 1982 00:01:01 GMT"
4242
may8_iso = "1982-05-08T00:01:01+00:00"
4343

44-
changes_url = server_url + RECORDS_URL.format("monitor", "changes")
44+
changes_url = server_url + CHANGESET_URL.format("monitor", "changes")
4545
mock_responses.get(
4646
changes_url,
4747
payload={
48-
"data": [
48+
"changes": [
4949
{
5050
"id": "abc",
5151
"bucket": "main",

‎tests/checks/remotesettings/test_attachments_integrity.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from checks.remotesettings.attachments_integrity import run
22

33

4+
CHANGESET_URL = "/buckets/{}/collections/{}/changeset"
45
RECORDS_URL = "/buckets/{}/collections/{}/records"
56

67

@@ -10,11 +11,11 @@ async def test_positive(mock_responses, mock_aioresponses):
1011
server_url + "/",
1112
payload={"capabilities": {"attachments": {"base_url": "http://cdn/"}}},
1213
)
13-
changes_url = server_url + RECORDS_URL.format("monitor", "changes")
14+
changes_url = server_url + CHANGESET_URL.format("monitor", "changes")
1415
mock_responses.get(
1516
changes_url,
1617
payload={
17-
"data": [
18+
"changes": [
1819
{"id": "abc", "bucket": "bid", "collection": "cid", "last_modified": 42}
1920
]
2021
},
@@ -59,11 +60,11 @@ async def test_negative(mock_responses, mock_aioresponses):
5960
server_url + "/",
6061
payload={"capabilities": {"attachments": {"base_url": "http://cdn/"}}},
6162
)
62-
changes_url = server_url + RECORDS_URL.format("monitor", "changes")
63+
changes_url = server_url + CHANGESET_URL.format("monitor", "changes")
6364
mock_responses.get(
6465
changes_url,
6566
payload={
66-
"data": [
67+
"changes": [
6768
{"id": "abc", "bucket": "bid", "collection": "cid", "last_modified": 42}
6869
]
6970
},

0 commit comments

Comments
 (0)