Add ability to mark releases as "required" (#122)

* handle "required" updates in Client.check_for_updates

* add "required" arg to Repository.add_bundle()

* warn if bundle is not added due to version

* make app version 2.0 required in repo workflow example, and update test data and tests accordingly

* update test data and simplify KEY_REQUIRED value

* use subparser names in cli commands

* add targets add --required CLI option

examples/repo/repo_workflow_example.py

@@ -90,7 +90,7 @@
 KEY_MAP['root'].append('root_two') # use two keys for root
 ENCRYPTED_KEYS = ['root', 'root_two', 'targets']
 
-# Custom metadata
+# Custom metadata (for example, a list of changes)
 DUMMY_METADATA = dict(changes=['this has changed', 'that has changed', '...'])
 
 # Create repository instance
@@ -169,7 +169,11 @@
  repo.add_bundle(
  new_version=new_version,
  new_bundle_dir=dummy_bundle_dir,
- custom_metadata=DUMMY_METADATA, # just to point out the option
+ # example of optional custom metadata
+ custom_metadata=DUMMY_METADATA.copy(),
+ # "required" updates are exceptional and should be avoided if possible,
+ # but we include one here just for completeness
+ required=new_version == '2.0',
  )
  repo.publish_changes(private_key_dirs=[OFFLINE_DIR_1, OFFLINE_DIR_2, ONLINE_DIR])
 

src/tufup/client.py

@@ -12,7 +12,7 @@
 from tuf.api.exceptions import DownloadError, UnsignedMetadataError
 import tuf.ngclient
 
-from tufup.common import Patcher, TargetMeta
+from tufup.common import KEY_REQUIRED, Patcher, TargetMeta
 from tufup.utils.platform_specific import install_update
 
 logger = logging.getLogger(__name__)
@@ -138,7 +138,10 @@ def download_and_apply_update(
  )
 
  def check_for_updates(
- self, pre: Optional[str] = None, patch: bool = True
+ self,
+ pre: Optional[str] = None,
+ patch: bool = True,
+ ignore_required: bool = False,
  ) -> Optional[TargetMeta]:
  """
  Check if any updates are available, based on current app version.
@@ -152,6 +155,14 @@ def check_for_updates(
  candidate, respectively.
 
  If `patch` is `False`, a full update is enforced.
+
+ If a new release is marked as "required" (in its custom metadata) this
+ release will take precedence over any non-required releases, *even* if the
+ latter are newer. This may be useful e.g. in case of a configuration change.
+ These "required" releases should be rare, and should preferably be avoided.
+ However, in the exceedingly rare event that there *are* "required" updates,
+ yet the user wants to treat them as non-required, they can specify
+ `ignore_required=True`.
  """
  # invalid pre-release specifiers are ignored, with a warning
  pre_map = dict(a='abrc', b='brc', rc='rc')
@@ -186,7 +197,16 @@ def check_for_updates(
  new_archive_meta = None
  if new_archives:
  logger.debug(f'{len(new_archives)} new *archives* found')
- new_archive_meta, self.new_archive_info = sorted(new_archives.items())[-1]
+ # the "latest" archive is typically just the last one in the sorted list
+ # of new archives, except when there are new "required" archives,
+ # in which case we must update to the first "required" archive encountered
+ for archive_meta, archive_info in sorted(new_archives.items()):
+ if not ignore_required and archive_meta.custom_internal:
+ if archive_meta.custom_internal.get(KEY_REQUIRED):
+ logger.debug(f'required update found: {archive_meta.version}')
+ break
+ new_archive_meta = archive_meta # noqa
+ self.new_archive_info = archive_info # noqa
  self.new_archive_local_path = pathlib.Path(
  self.target_dir, new_archive_meta.path.name
  )

src/tufup/common.py

@@ -10,6 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
+KEY_REQUIRED = 'required' # unlikely to be identical to user-specified key
 SUFFIX_ARCHIVE = '.tar.gz'
 SUFFIX_PATCH = '.patch'
 
@@ -19,6 +20,7 @@ class CustomMetadataDict(TypedDict):
  explicitly separate custom metadata into user-specified metadata and metadata
  used by tufup internally
  """
+
  user: Optional[dict]
  tufup: Optional[dict]
 

src/tufup/repo/__init__.py

@@ -38,7 +38,13 @@
 )
 from tuf.api.serialization.json import JSONSerializer
 
-from tufup.common import CustomMetadataDict, Patcher, SUFFIX_PATCH, TargetMeta
+from tufup.common import (
+ CustomMetadataDict,
+ KEY_REQUIRED,
+ Patcher,
+ SUFFIX_PATCH,
+ TargetMeta,
+)
 from tufup.utils.platform_specific import _patched_resolve
 
 logger = logging.getLogger(__name__)
@@ -735,6 +741,7 @@ def add_bundle(
  new_version: Optional[str] = None,
  skip_patch: bool = False,
  custom_metadata: Optional[dict] = None, # archive only
+ required: bool = False,
  ):
  """
  Adds a new application bundle to the local repository.
@@ -744,6 +751,12 @@ def add_bundle(
  a patch file is also created and added to the repository, unless
  `skip_patch` is True.
 
+ If `required=True` (default is `False`), this release will always be
+ installed, even if newer releases are available. For example, suppose
+ an app is running at version 1.0, and version 2.0 is required, but version
+ 3.0 is also available, then tufup will first update to version 2.0,
+ before updating to 3.0 on the next run.
+
  Note the changes are not published yet: call `publish_changes()` for
  that.
  """
@@ -769,7 +782,7 @@ def add_bundle(
  self.roles.add_or_update_target(
  local_path=new_archive.path,
  # separate user-specified metadata from tufup-internal metadata
- custom=dict(user=custom_metadata, tufup=None),
+ custom=dict(user=custom_metadata, tufup={KEY_REQUIRED: required}),
  )
  # create patch, if possible, and register that too
  if latest_archive and not skip_patch:
@@ -786,6 +799,11 @@ def add_bundle(
  local_path=patch_path,
  custom=dict(user=None, tufup=dst_size_and_hash),
  )
+ else:
+ logger.warning(
+ f'bundle not added: version {new_archive.version} must be greater than'
+ f'that of latest archive ({latest_archive.version})'
+ )
 
  def remove_latest_bundle(self):
  """

src/tufup/repo/cli.py

@@ -18,6 +18,7 @@
  targets_add_app_version='Application version (PEP440 compliant)',
  targets_add_bundle_dir='Directory containing application bundle.',
  targets_add_skip_patch='Skip patch creation.',
+ targets_add_required='Mark release as "required".',
  targets_remove_latest='Remove latest app bundle from the repository.',
  keys_subcommands='Optional commands to add or replace keys.',
  keys_new_key_name='Name of new private key (public key gets .pub suffix).',
@@ -63,7 +64,9 @@ def get_parser() -> argparse.ArgumentParser:
  subparser_targets = subparsers.add_parser('targets', parents=[debug_parser])
  subparser_targets.set_defaults(func=_cmd_targets)
  # we use nested subparsers to deal with mutually dependent arguments
- targets_subparsers = subparser_targets.add_subparsers()
+ targets_subparsers = subparser_targets.add_subparsers(
+ dest='subcommand', required=True
+ )
  subparser_targets_add = targets_subparsers.add_parser(
  'add', help=HELP['targets_add']
  )
@@ -82,6 +85,13 @@ def get_parser() -> argparse.ArgumentParser:
  required=False,
  help=HELP['targets_add_skip_patch'],
  )
+ subparser_targets_add.add_argument(
+ '-r',
+ '--required',
+ action='store_true',
+ required=False,
+ help=HELP['targets_add_required'],
+ )
  subparser_targets_remove = targets_subparsers.add_parser(
  'remove-latest', help=HELP['targets_remove_latest']
  )
@@ -98,7 +108,9 @@ def get_parser() -> argparse.ArgumentParser:
  '-e', '--encrypted', action='store_true', help=HELP['keys_encrypted']
  )
  # we use nested subparsers to deal with mutually dependent arguments
- keys_subparsers = subparser_keys.add_subparsers(help=HELP['keys_subcommands'])
+ keys_subparsers = subparser_keys.add_subparsers(
+ dest='subcommand', help=HELP['keys_subcommands']
+ )
  subparser_keys_add = keys_subparsers.add_parser('add')
  subparser_keys_add.add_argument(
  'role_name', choices=TOP_LEVEL_ROLE_NAMES, help=HELP['keys_role_name']
@@ -237,9 +249,7 @@ def _cmd_keys(options: argparse.Namespace):
  private_key_path=private_key_path, encrypted=options.encrypted
  )
  _print_info('Key pair created.')
- replace = hasattr(options, 'old_key_name')
- add = hasattr(options, 'role_name')
- if replace:
+ if options.subcommand == 'replace':
  _print_info(
  f'Replacing key {options.old_key_name} by {options.new_key_name}...'
  )
@@ -248,14 +258,14 @@ def _cmd_keys(options: argparse.Namespace):
  new_public_key_path=public_key_path,
  new_private_key_encrypted=options.encrypted,
  )
- elif add:
+ elif options.subcommand == 'add':
  _print_info(f'Adding key {options.new_key_name}...')
  repository.add_key(
  role_name=options.role_name,
  public_key_path=public_key_path,
  encrypted=options.encrypted,
  )
- if replace or add:
+ if options.subcommand in ['add', 'replace']:
  _print_info('Publishing changes...')
  repository.publish_changes(private_key_dirs=options.key_dirs)
  _print_info('Done.')
@@ -264,14 +274,15 @@ def _cmd_keys(options: argparse.Namespace):
 def _cmd_targets(options: argparse.Namespace):
  logger.debug(f'command targets: {vars(options)}')
  repository = _get_repo()
- if hasattr(options, 'app_version') and hasattr(options, 'bundle_dir'):
+ if options.subcommand == 'add':
  _print_info('Adding bundle...')
  repository.add_bundle(
  new_version=options.app_version,
  new_bundle_dir=options.bundle_dir,
  skip_patch=options.skip_patch,
+ required=options.required,
  )
- else:
+ elif options.subcommand == 'remove-latest':
  _print_info('Removing latest bundle...')
  repository.remove_latest_bundle()
  _print_info('Publishing changes...')

tests/data/repository/metadata/1.root.json

@@ -2,17 +2,17 @@
  "signatures": [
  {
  "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "sig": "9f80547ca0ab37b5de2ae3869be83b9e1312636c3f932265e1e02f528cd59c5057dee7f7d76f7d4f19241538fc578cd01c15b02e9ecfd9a2b6dd1324a53a0008"
+ "sig": "6970cde311f0bdf21335dbb80d1500215febf441b74e800445f25810f48b503f962eea2d314865336ccc83aae308d493707d2eca0f852496a4c1555e9eb7d301"
  },
  {
  "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
- "sig": "68a0d73c4327f7ac39dd15476e8d5d11fdedd034e5795e7ee1d4bea0066046e7d07f1508e2df0b7f99f39d66dc3c2b540632bafc121dbfa4d1c43f6f07448504"
+ "sig": "4977d4109d9e7ba2a51e34988c5c45fe0a7e81b0edc64d0de428b362ca8397056cb168c85c764d44dc63124e085d7bfa9907bb702c656040bc912bcc1ec1ac0a"
  }
  ],
  "signed": {
  "_type": "root",
  "consistent_snapshot": false,
- "expires": "2051-07-25T14:59:45Z",
+ "expires": "2051-07-25T16:49:47Z",
  "keys": {
  "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": {
  "keytype": "ed25519",

tests/data/repository/metadata/2.root.json

@@ -2,21 +2,21 @@
  "signatures": [
  {
  "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
- "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
+ "sig": "b3edc83b2236ccd0c78882e36431d6f2cfc1ae187d71cdae57864f18608d3b2da0495f843a173090aa0483d61e800e60eb00c55b5478b6aa7cef890cf67a1800"
  },
  {
  "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
+ "sig": "e70fba716bac644045b1e76f378e601dd56884ab8976f50cf2076da4643e3de1aeff2fc48b6c19336db39e735ce7dce526f9445d0c11b1331398355cf2929602"
  },
  {
  "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
- "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
+ "sig": "7ab48e365e9ddf4559abf719917fe7a49dbc5e30f5077c6f69d2d4db6e9352b53d5930b70970c70bb5570f2cbeb0ab0ba236c75e9209f689f073633708aeea09"
  }
  ],
  "signed": {
  "_type": "root",
  "consistent_snapshot": false,
- "expires": "2051-07-25T14:59:52Z",
+ "expires": "2051-07-25T16:49:51Z",
  "keys": {
  "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
  "keytype": "ed25519",

tests/data/repository/metadata/root.json

@@ -2,21 +2,21 @@
  "signatures": [
  {
  "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
- "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
+ "sig": "b3edc83b2236ccd0c78882e36431d6f2cfc1ae187d71cdae57864f18608d3b2da0495f843a173090aa0483d61e800e60eb00c55b5478b6aa7cef890cf67a1800"
  },
  {
  "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
- "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
+ "sig": "e70fba716bac644045b1e76f378e601dd56884ab8976f50cf2076da4643e3de1aeff2fc48b6c19336db39e735ce7dce526f9445d0c11b1331398355cf2929602"
  },
  {
  "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
- "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
+ "sig": "7ab48e365e9ddf4559abf719917fe7a49dbc5e30f5077c6f69d2d4db6e9352b53d5930b70970c70bb5570f2cbeb0ab0ba236c75e9209f689f073633708aeea09"
  }
  ],
  "signed": {
  "_type": "root",
  "consistent_snapshot": false,
- "expires": "2051-07-25T14:59:52Z",
+ "expires": "2051-07-25T16:49:51Z",
  "keys": {
  "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
  "keytype": "ed25519",

tests/data/repository/metadata/snapshot.json

@@ -2,12 +2,12 @@
  "signatures": [
  {
  "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a",
- "sig": "48bdd9a911dc60620c513706bd0140573611a85713844d7ec38d938126ad4088688a6829d819049cd94c8a01ed1448e707800f7b4438a7edaa9edd1919612501"
+ "sig": "5b6c5ce5c2333b93a8f0e86b68fca178b301ee33db6db1df3ed9620e738c9b097886a5b8b8c7b297083f91ca3ca30167e6d9f576e033f01a36471a64bca22708"
  }
  ],
  "signed": {
  "_type": "snapshot",
- "expires": "2051-07-25T14:59:52Z",
+ "expires": "2051-07-25T16:49:51Z",
  "meta": {
  "targets.json": {
  "version": 6

tests/data/repository/metadata/targets.json

@@ -2,17 +2,19 @@
  "signatures": [
  {
  "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5",
- "sig": "ba92827c2fbe262ca8dc28dfc6b2629a4ce0c8af747d666b00e1104ccba004e3727a85eb00e957edeadae2a430972210c949723f4091e9f01f676025868bef03"
+ "sig": "af58034e8d5e5d525da096ad8946b44a2e82882d5a2f400b84e9ce956412d42c31f75a274b566dfe0f13117988a3fe605512e0924c4e9bd06bc33002b1b0c606"
  }
  ],
  "signed": {
  "_type": "targets",
- "expires": "2051-07-25T14:59:52Z",
+ "expires": "2051-07-25T16:49:51Z",
  "spec_version": "1.0.31",
  "targets": {
  "example_app-1.0.tar.gz": {
  "custom": {
- "tufup": null,
+ "tufup": {
+ "required": false
+ },
  "user": null
  },
  "hashes": {
@@ -36,7 +38,9 @@
  },
  "example_app-2.0.tar.gz": {
  "custom": {
- "tufup": null,
+ "tufup": {
+ "required": true
+ },
  "user": {
  "changes": [
  "this has changed",
@@ -66,7 +70,9 @@
  },
  "example_app-3.0rc0.tar.gz": {
  "custom": {
- "tufup": null,
+ "tufup": {
+ "required": false
+ },
  "user": {
  "changes": [
  "this has changed",
@@ -96,7 +102,9 @@
  },
  "example_app-4.0a0.tar.gz": {
  "custom": {
- "tufup": null,
+ "tufup": {
+ "required": false
+ },
  "user": {
  "changes": [
  "this has changed",

tests/data/repository/metadata/timestamp.json

@@ -2,12 +2,12 @@
  "signatures": [
  {
  "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00",
- "sig": "cbff18b83e93143a98a418dccea7f1d2f4eefe466a955860c3dedd730b8ac08fa01b4eb3f3c96a3cb943e8bca154d5ea52aacdb97e53ea4a96a8441e9145b90e"
+ "sig": "5e1b959bf5dea6612ac5ee08bd407379c51bfe9be7b9c87ff3f6c7ce2df3cc2508db3bc57abe9b7807e44485e2f02aaa6cdf511a1887425357ff25b2bb10ba09"
  }
  ],
  "signed": {
  "_type": "timestamp",
- "expires": "2051-07-25T14:59:53Z",
+ "expires": "2051-07-25T16:49:51Z",
  "meta": {
  "snapshot.json": {
  "version": 7