diff --git a/cnudie/purge.py b/cnudie/purge.py index 7c04c39ad..bc682da92 100644 --- a/cnudie/purge.py +++ b/cnudie/purge.py @@ -1,4 +1,3 @@ -import collections.abc import logging import traceback @@ -9,29 +8,9 @@ import cnudie.util import oci.client as oc import oci.model as om -import version -logger = logging.getLogger(__name__) - - -def iter_componentversions_to_purge( - component: ocm.Component | ocm.ComponentDescriptor, - policy: version.VersionRetentionPolicies, - oci_client: oc.Client, -) -> collections.abc.Generator[ocm.ComponentIdentity, None, None]: - oci_ref = cnudie.util.oci_ref(component=component) - if isinstance(component, ocm.ComponentDescriptor): - component = component.component - for v in version.versions_to_purge( - versions=oci_client.tags(oci_ref.ref_without_tag), - reference_version=component.version, - policy=policy, - ): - yield ocm.ComponentIdentity( - name=component.name, - version=v, - ) +logger = logging.getLogger(__name__) def remove_component_descriptor_and_referenced_artefacts( diff --git a/concourse/model/traits/component_descriptor.py b/concourse/model/traits/component_descriptor.py index 9688a0b2a..222f32cb8 100644 --- a/concourse/model/traits/component_descriptor.py +++ b/concourse/model/traits/component_descriptor.py @@ -12,11 +12,9 @@ from ci.util import not_none from ocm import Label -import ci.util import cnudie.retrieve import ocm import model.base -import version from concourse.model.job import ( JobVariant, @@ -89,60 +87,14 @@ class UploadMode(enum.StrEnum): AttributeSpec.optional( name='retention_policy', default=None, - type=typing.Union[version.VersionRetentionPolicies, str], - doc=''' - specifies how (old) component-descriptors and their referenced resources should be - handled. This is foremostly intended as an option for automated cleanup for - components with frequent (shortlived) releases and/or frequent (shortlived) snapshots. - - if no retention_policy is defined, no cleanup will be done. - - retention policy may either be defined "inline" (as a mapping value) or by referencing - a pre-defined policy by name (see `retentions_policies` attribute). In the latter case, - use the policie's name as (string) attribute value. - ''', + type=str, + doc='obsolete', ), AttributeSpec.optional( name='retention_policies', - default=[ - version.VersionRetentionPolicies( - name='clean-snapshots', - rules=[ - version.VersionRetentionPolicy( - name='clean-snapshots', - keep=64, - match=version.VersionType.SNAPSHOT, - ), - version.VersionRetentionPolicy( - name='keep-releases', - keep='all', - match=version.VersionType.RELEASE, - ), - ], - dry_run=False, - ), - version.VersionRetentionPolicies( - name='clean-snapshots-and-releases', - rules=[ - version.VersionRetentionPolicy( - name='clean-snapshots', - keep=64, - match=version.VersionType.SNAPSHOT, - ), - version.VersionRetentionPolicy( - name='clean-releases', - keep=128, - match=version.VersionType.RELEASE, - ), - ], - dry_run=False, - ), - ], - type=typing.List[version.VersionRetentionPolicies], - doc=''' - predefined retention policies (see default value). may be referenced via - `retention_policy` attribute (adding additional policies here has no immediate effect) - ''' + default=[], + type=typing.List[str], + doc='obsolete' ), AttributeSpec.deprecated( name='validation_policies', @@ -229,45 +181,6 @@ def step_name(self): def upload(self) -> UploadMode: return UploadMode(self.raw['upload']) - def retention_policy(self, raw=True) -> version.VersionRetentionPolicies | None: - if not (policy := self.raw.get('retention_policy', None)): - return None - - if isinstance(policy, str): - # lookup name - for candidate in self.raw.get('retention_policies', ()): - if isinstance(candidate, dict): - name = candidate['name'] - elif isinstance(candidate, version.VersionRetentionPolicies): - name = candidate.name - else: - raise ValueError(candidate) - - if name == policy: - policy = candidate - break - else: - raise ValueError(f'did not find {policy=} in retention_policies') - - if raw: - if isinstance(policy, version.VersionRetentionPolicies): - policy = dataclasses.asdict( - policy, - dict_factory=ci.util.dict_factory_enum_serialisiation, - ) - return policy - - if isinstance(policy, version.VersionRetentionPolicies): - return policy - - return dacite.from_dict( - data_class=version.VersionRetentionPolicies, - data=policy, - config=dacite.Config( - cast=(enum.Enum,), - ), - ) - def resolve_dependencies(self): return self.raw['resolve_dependencies'] diff --git a/concourse/steps/component_descriptor.mako b/concourse/steps/component_descriptor.mako index dda709fe0..592cdf735 100644 --- a/concourse/steps/component_descriptor.mako +++ b/concourse/steps/component_descriptor.mako @@ -24,7 +24,6 @@ if descriptor_trait.ocm_repository: ocm_repository_url = descriptor_trait.ocm_repository.oci_ref else: ocm_repository_url = None -retention_policy = descriptor_trait.retention_policy() ocm_repository_mappings = descriptor_trait.ocm_repository_mappings() # label main repo as main @@ -333,60 +332,6 @@ else: ) with open(dependencies_path) as f: print(f.read()) -% if retention_policy: - -logger.info('will honour retention-policy') -retention_policy = dacite.from_dict( - data_class=version.VersionRetentionPolicies, - data=${retention_policy}, - config=dacite.Config(cast=(enum.Enum,)), -) -pprint.pprint(retention_policy) - -if retention_policy.dry_run: - logger.info('dry-run - will only print versions to remove, but not actually remove them') -else: - logger.info('!! will attempt to remove listed component-versions, according to policy') - -logger.info('the following versions were identified for being purged') -component = descriptor_v2.component - - -for idx, component_id in enumerate(cnudie.purge.iter_componentversions_to_purge( - component=component, - policy=retention_policy, - oci_client=oci_client, -)): - if idx >= 64: - print('will abort the purge, considering there seem to be more than 64 versions to cleanup') - print('this is done to limit execution-time - the purge will continue on next execution') - exit(0) - print(f'{idx} {component_id.name}:{component_id.version}') - if retention_policy.dry_run: - continue - component_to_purge = component_descriptor_lookup( - ocm.ComponentIdentity( - name=component.name, - version=component_id.version, - ) - ) - if not component_to_purge: - logger.warning(f'{component.name}:{component_id.version} was not found - ignore') - continue - - try: - cnudie.purge.remove_component_descriptor_and_referenced_artefacts( - component=component_to_purge, - oci_client=oci_client, - lookup=component_descriptor_lookup, - recursive=False, - ) - except Exception as e: - logger.warning(f'error occurred while trying to purge {component_id}: {e}') - traceback.print_exc() -% else: -logger.info('no retention-policy was defined - will not purge component-descriptors') -% endif <%def diff --git a/doc/traits/component_descriptor.rst b/doc/traits/component_descriptor.rst index 59afc3b5b..9749c8464 100644 --- a/doc/traits/component_descriptor.rst +++ b/doc/traits/component_descriptor.rst @@ -205,20 +205,6 @@ Gardener-Components have a name that is by convention the github-repo-url (w/o s - name: imagevector.gardener.cloud/source-repository value: github.com/gardener/gardener # github-repo - -Retention Policies (aka cleaning up old versions) -================================================= - -The `retention_policies` attribute can be used to configure automated removal of -component descriptors and referenced `resources` (mostly OCI Container Images). - -.. attention:: - Removal of component descriptors and referenced resources is _permanent_. There is no - backup mechanism in place. Use with care. For example, if multiple component descriptors - share reference to the same OCI Artefact (using the same registry, repository, and tag) - removal of any of the referencing component descriptors will lead to stale references - in other component descriptors. - Cleanup Semantics and Use-Case ------------------------------ diff --git a/version.py b/version.py index d4b5c96be..02ee0d702 100644 --- a/version.py +++ b/version.py @@ -4,7 +4,6 @@ import collections -import dataclasses import enum import logging import semver @@ -41,47 +40,6 @@ class VersionType(enum.Enum): ANY = 'any' -@dataclasses.dataclass(frozen=True) -class VersionRetentionPolicy: - name: str = None - keep: typing.Union[str, int] = 'all' - match: VersionType = VersionType.ANY - restrict: VersionRestriction = VersionRestriction.NONE - recursive: bool = False - - def matches_version_restriction(self, version, ref_version) -> bool: - version = parse_to_semver(version) - final = is_final(version) - - if self.match is VersionType.ANY: - pass - if self.match is VersionType.SNAPSHOT and final: - return False - if self.match is VersionType.RELEASE and not final: - return False - - # if this line is reached, version-type matches - - if self.restrict is VersionRestriction.NONE: - return True - elif self.restrict is VersionRestriction.SAME_MINOR: - ref_version = version.parse_to_semver(ref_version) - return ref_version.minor == version.minor - else: - raise RuntimeError(f'not implemented: {self.restrict}') - - @property - def keep_all(self) -> bool: - return self.keep == 'all' - - -@dataclasses.dataclass -class VersionRetentionPolicies: - name: str - rules: list[VersionRetentionPolicy] - dry_run: bool = True - - T = typing.TypeVar('T') @@ -95,46 +53,6 @@ def is_final( return not version.build and not version.prerelease -def versions_to_purge( - versions: typing.Iterable[T], - reference_version: Version, - policy: VersionRetentionPolicies, - converter: typing.Callable[[T], Version]=None, -) -> typing.Generator[T, None, None]: - versions_by_policy = collections.defaultdict(list) - - def to_version(v: T): - if converter: - v = converter(v) - return v - - for v in versions: - converted_version = to_version(v) - for rule in policy.rules: - rule: VersionRetentionPolicy - if rule.matches_version_restriction( - version=converted_version, - ref_version=reference_version, - ): - versions_by_policy[rule].append(v) - break # first rule matches - else: - continue - else: - logger.info(f'no rule matched {converted_version}') - - for policy, versions in versions_by_policy.items(): - policy: VersionRetentionPolicy - if policy.keep_all: - continue - - yield from smallest_versions( - versions=versions, - keep=policy.keep, - converter=converter, - ) - - def parse_to_semver( version, invalid_semver_ok: bool=False,