Separate custom metadata into user-specified and tufup-internal (#123)

* separate user-specified custom metadata from tufup-internal custom metadata (backward compatible: can read older metadata that does not make the distinction) * update test data and test custom metadata separation
dennisvang · Mar 8, 2024 · 4e02bfe · 4e02bfe
1 parent cbf8226
commit 4e02bfe
Show file tree

Hide file tree

Showing 10 changed files with 134 additions and 58 deletions.
diff --git a/src/tufup/common.py b/src/tufup/common.py
@@ -3,7 +3,7 @@
 import logging
 import pathlib
 import re
-from typing import Dict, Optional, Union
+from typing import Dict, get_type_hints, Optional, TypedDict, Union
 
 import bsdiff4
 from packaging.version import Version, InvalidVersion
@@ -14,6 +14,15 @@
 SUFFIX_PATCH = '.patch'
 
 
+class CustomMetadataDict(TypedDict):
+ """
+ explicitly separate custom metadata into user-specified metadata and metadata
+ used by tufup internally
+ """
+ user: Optional[dict]
+ tufup: Optional[dict]
+
+
 def _immutable(value):
  """
  Make value immutable, recursively, so the result is hashable.
@@ -49,7 +58,7 @@ def __init__(
  name: Optional[str] = None,
  version: Optional[str] = None,
  is_archive: Optional[bool] = True,
- custom: Optional[dict] = None,
+ custom: Optional[CustomMetadataDict] = None,
  ):
  """
  Initialize either with target_path, or with name, version, archive.
@@ -68,7 +77,28 @@ def __init__(
  logger.critical(
  f'invalid filename "{self.filename}": whitespace not allowed'
  )
- self.custom = custom
+ self._custom = custom
+
+ @property
+ def custom(self) -> Optional[dict]:
+ """returns user-specified custom metadata"""
+ return self._get_custom_metadata('user')
+
+ @property
+ def custom_internal(self) -> Optional[dict]:
+ """returns tufup-internal custom metadata"""
+ return self._get_custom_metadata('tufup')
+
+ def _get_custom_metadata(self, key: str) -> Optional[dict]:
+ """
+ get custom metadata in a backward-compatible manner (older versions did not
+ distinguish between user-specified and internal metadata)
+ """
+ if isinstance(self._custom, dict):
+ # check dict keys for backward compatibility
+ if get_type_hints(CustomMetadataDict).keys() != self._custom.keys():
+ return self._custom
+ return self._custom.get(key)
 
  def __str__(self):
  return str(self.target_path_str)
@@ -246,7 +276,7 @@ def patch_and_verify(
  # verify integrity of the final result (raises exception on failure)
  cls._verify_tar_size_and_hash(
  tar_content=tar_bytes,
- expected=patch_meta.custom, # noqa
+ expected=patch_meta.custom_internal, # noqa
  )
  # compress .tar data into destination .tar.gz file
  with gzip.open(dst_path, mode='wb') as dst_file:

diff --git a/src/tufup/repo/__init__.py b/src/tufup/repo/__init__.py
@@ -38,7 +38,7 @@
 )
 from tuf.api.serialization.json import JSONSerializer
 
-from tufup.common import Patcher, SUFFIX_PATCH, TargetMeta
+from tufup.common import CustomMetadataDict, Patcher, SUFFIX_PATCH, TargetMeta
 from tufup.utils.platform_specific import _patched_resolve
 
 logger = logging.getLogger(__name__)
@@ -375,7 +375,7 @@ def add_or_update_target(
  self,
  local_path: Union[pathlib.Path, str],
  url_path_segments: Optional[List[str]] = None,
- custom: Optional[dict] = None,
+ custom: Optional[CustomMetadataDict] = None,
  ):
  # based on python-tuf basic_repo.py
  local_path = pathlib.Path(local_path)
@@ -767,7 +767,9 @@ def add_bundle(
  if not latest_archive or latest_archive.version < new_archive.version:
  # register new archive
  self.roles.add_or_update_target(
- local_path=new_archive.path, custom=custom_metadata
+ local_path=new_archive.path,
+ # separate user-specified metadata from tufup-internal metadata
+ custom=dict(user=custom_metadata, tufup=None),
  )
  # create patch, if possible, and register that too
  if latest_archive and not skip_patch:
@@ -781,7 +783,8 @@ def add_bundle(
  # register patch (size and hash are used by the client to verify the
  # integrity of the patched archive)
  self.roles.add_or_update_target(
- local_path=patch_path, custom=dst_size_and_hash
+ local_path=patch_path,
+ custom=dict(user=None, tufup=dst_size_and_hash),
  )
 
  def remove_latest_bundle(self):

diff --git a/tests/data/repository/metadata/1.root.json b/tests/data/repository/metadata/1.root.json
@@ -1,18 +1,18 @@
 {
  "signatures": [
  {
- "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
- "sig": "0f634a6e5f82af4447accce63c2987350c9c16fe6f8ce391ed504da106be8a127e1d606424c97a27822038cfd35e4daa96da2ec07a4a75bc2610df3bfc95cd0c"
+ "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
+ "sig": "9f80547ca0ab37b5de2ae3869be83b9e1312636c3f932265e1e02f528cd59c5057dee7f7d76f7d4f19241538fc578cd01c15b02e9ecfd9a2b6dd1324a53a0008"
  },
  {
- "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "sig": "678256d67bcf6022f75920ff380dc2111e2d68120af834f1769d694665236a2c7fb57ea5731f4050e1562a8b2be870b6594a2203f52182b1b77fa98ae89ed90c"
+ "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+ "sig": "68a0d73c4327f7ac39dd15476e8d5d11fdedd034e5795e7ee1d4bea0066046e7d07f1508e2df0b7f99f39d66dc3c2b540632bafc121dbfa4d1c43f6f07448504"
  }
  ],
  "signed": {
  "_type": "root",
  "consistent_snapshot": false,
- "expires": "2051-06-27T21:21:03Z",
+ "expires": "2051-07-25T14:59:45Z",
  "keys": {
  "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": {
  "keytype": "ed25519",
@@ -53,8 +53,8 @@
  "roles": {
  "root": {
  "keyids": [
- "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568"
+ "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+ "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2"
  ],
  "threshold": 2
  },

diff --git a/tests/data/repository/metadata/2.root.json b/tests/data/repository/metadata/2.root.json
@@ -2,21 +2,21 @@
  "signatures": [
  {
  "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
- "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503"
+ "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
  },
  {
- "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
- "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005"
+ "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
+ "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
  },
  {
- "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00"
+ "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+ "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
  }
  ],
  "signed": {
  "_type": "root",
  "consistent_snapshot": false,
- "expires": "2051-06-27T21:21:13Z",
+ "expires": "2051-07-25T14:59:52Z",
  "keys": {
  "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
  "keytype": "ed25519",

diff --git a/tests/data/repository/metadata/root.json b/tests/data/repository/metadata/root.json
@@ -2,21 +2,21 @@
  "signatures": [
  {
  "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
- "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503"
+ "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
  },
  {
- "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
- "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005"
+ "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
+ "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
  },
  {
- "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00"
+ "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+ "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
  }
  ],
  "signed": {
  "_type": "root",
  "consistent_snapshot": false,
- "expires": "2051-06-27T21:21:13Z",
+ "expires": "2051-07-25T14:59:52Z",
  "keys": {
  "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
  "keytype": "ed25519",

diff --git a/tests/data/repository/metadata/snapshot.json b/tests/data/repository/metadata/snapshot.json
@@ -2,12 +2,12 @@
  "signatures": [
  {
  "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a",
- "sig": "73a146f5e1f12c0a36e88c8d7bf613baa1d528ea0c9480fe0d2ccd74d6da239da04470f68d283738194185cc82289c5f9f1312efea373b51dc8722965ca1fc0b"
+ "sig": "48bdd9a911dc60620c513706bd0140573611a85713844d7ec38d938126ad4088688a6829d819049cd94c8a01ed1448e707800f7b4438a7edaa9edd1919612501"
  }
  ],
  "signed": {
  "_type": "snapshot",
- "expires": "2051-06-27T21:21:13Z",
+ "expires": "2051-07-25T14:59:52Z",
  "meta": {
  "targets.json": {
  "version": 6

diff --git a/tests/data/repository/metadata/targets.json b/tests/data/repository/metadata/targets.json
@@ -2,25 +2,32 @@
  "signatures": [
  {
  "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5",
- "sig": "344b1c779103db5c8462508d7a5e72ef9ae8dea0c5fd303d55cace03a87fd67312ff5ca01fc2e377d7d0dcbbbf3f4dff378f5c9759801590340c0b9e3d23bc07"
+ "sig": "ba92827c2fbe262ca8dc28dfc6b2629a4ce0c8af747d666b00e1104ccba004e3727a85eb00e957edeadae2a430972210c949723f4091e9f01f676025868bef03"
  }
  ],
  "signed": {
  "_type": "targets",
- "expires": "2051-06-27T21:21:13Z",
+ "expires": "2051-07-25T14:59:52Z",
  "spec_version": "1.0.31",
  "targets": {
  "example_app-1.0.tar.gz": {
+ "custom": {
+ "tufup": null,
+ "user": null
+ },
  "hashes": {
  "sha256": "223dfd468edbe36256dc119f8477ac4025279ae3705dec04a32002e81b10fd16"
  },
  "length": 101613
  },
  "example_app-2.0.patch": {
  "custom": {
- "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5",
- "tar_hash_algorithm": "sha256",
- "tar_size": 112640
+ "tufup": {
+ "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5",
+ "tar_hash_algorithm": "sha256",
+ "tar_size": 112640
+ },
+ "user": null
  },
  "hashes": {
  "sha256": "f2be4504e464bd23c022772c7f3c011e0082295775a24e3fc986bb2504df0f53"
@@ -29,11 +36,14 @@
  },
  "example_app-2.0.tar.gz": {
  "custom": {
- "changes": [
- "this has changed",
- "that has changed",
- "..."
- ]
+ "tufup": null,
+ "user": {
+ "changes": [
+ "this has changed",
+ "that has changed",
+ "..."
+ ]
+ }
  },
  "hashes": {
  "sha256": "d85f423a56427e522ac4d093a6ce94abcc2cc32f99b80a39b87832d8e4ba9ad8"
@@ -42,9 +52,12 @@
  },
  "example_app-3.0rc0.patch": {
  "custom": {
- "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5",
- "tar_hash_algorithm": "sha256",
- "tar_size": 112640
+ "tufup": {
+ "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5",
+ "tar_hash_algorithm": "sha256",
+ "tar_size": 112640
+ },
+ "user": null
  },
  "hashes": {
  "sha256": "01fa6f30ac54fd405dfb5bd6e39f71af572c8e341675f5b2822b35ec341ce6f9"
@@ -53,11 +66,14 @@
  },
  "example_app-3.0rc0.tar.gz": {
  "custom": {
- "changes": [
- "this has changed",
- "that has changed",
- "..."
- ]
+ "tufup": null,
+ "user": {
+ "changes": [
+ "this has changed",
+ "that has changed",
+ "..."
+ ]
+ }
  },
  "hashes": {
  "sha256": "d7fa6ddd397282e8fa81924a31f340ebbcb8c082604c0549a38f5882cd3716c6"
@@ -66,9 +82,12 @@
  },
  "example_app-4.0a0.patch": {
  "custom": {
- "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45",
- "tar_hash_algorithm": "sha256",
- "tar_size": 112640
+ "tufup": {
+ "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45",
+ "tar_hash_algorithm": "sha256",
+ "tar_size": 112640
+ },
+ "user": null
  },
  "hashes": {
  "sha256": "e19ccd94e60d1d817dcc44c481f4fb48f54fa335cc0bd7a7e60377a4f6ccf2ea"
@@ -77,11 +96,14 @@
  },
  "example_app-4.0a0.tar.gz": {
  "custom": {
- "changes": [
- "this has changed",
- "that has changed",
- "..."
- ]
+ "tufup": null,
+ "user": {
+ "changes": [
+ "this has changed",
+ "that has changed",
+ "..."
+ ]
+ }
  },
  "hashes": {
  "sha256": "05304765a0cb4e40cbd7ca2587aeb5f0db36cd9b617921e0d141bb91d3304e2c"

diff --git a/tests/data/repository/metadata/timestamp.json b/tests/data/repository/metadata/timestamp.json
@@ -2,12 +2,12 @@
  "signatures": [
  {
  "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00",
- "sig": "4f05f9947e1fd704ffb877fa994e8841eb1a34a2ccf021c2eac5a618eca3c11baad8531d154fa5c8aa187e0bfaa57b521901ef502bd7a3dc601bfa1c408a4106"
+ "sig": "cbff18b83e93143a98a418dccea7f1d2f4eefe466a955860c3dedd730b8ac08fa01b4eb3f3c96a3cb943e8bca154d5ea52aacdb97e53ea4a96a8441e9145b90e"
  }
  ],
  "signed": {
  "_type": "timestamp",
- "expires": "2051-06-27T21:21:13Z",
+ "expires": "2051-07-25T14:59:53Z",
  "meta": {
  "snapshot.json": {
  "version": 7

diff --git a/tests/test_client.py b/tests/test_client.py
@@ -112,7 +112,7 @@ def test_trusted_target_metas(self):
  self.assertIn(example_key, meta.custom)
  else:
  # patches must have tar hash information
- self.assertIn('tar_hash', meta.custom)
+ self.assertIn('tar_hash', meta.custom_internal)
 
  def test_get_targetinfo(self):
  client = self.get_refreshed_client()

diff --git a/tests/test_common.py b/tests/test_common.py
@@ -153,6 +153,27 @@ def test_compose_filename(self):
  )
  self.assertEqual('app-1.0.tar.gz', filename)
 
+ def test_custom_metadata(self):
+ user_metadata = dict(foo='bar')
+ internal_metadata = dict(something=True)
+ target_meta = TargetMeta(
+ custom=dict(user=user_metadata, tufup=internal_metadata)
+ )
+ self.assertEqual(user_metadata, target_meta.custom)
+ self.assertEqual(internal_metadata, target_meta.custom_internal)
+
+ def test_custom_metadata_backward_compatibility(self):
+ # older versions of tufup did not distinguish between user and internal metadata
+ custom_metadata = dict(foo='bar')
+ target_meta = TargetMeta(custom=custom_metadata) # noqa
+ self.assertEqual(custom_metadata, target_meta.custom)
+ self.assertEqual(custom_metadata, target_meta.custom_internal)
+
+ def test_custom_metadata_not_specified(self):
+ target_meta = TargetMeta()
+ self.assertIsNone(target_meta.custom)
+ self.assertIsNone(target_meta.custom_internal)
+
 
 class PatcherTests(TempDirTestCase):
  def setUp(self) -> None: