Skip to content

Commit

Permalink
Rm ocm retention policy
Browse files Browse the repository at this point in the history 
Google Artifact Registry has a built-in mechanism.
  • Loading branch information
@zkdev
zkdev committed Feb 27, 2025
1 parent f2ee3a5 commit 8377c7e
Show file tree
Hide file tree
Showing 5 changed files with 6 additions and 265 deletions.
23 changes: 1 addition & 22 deletions cnudie/purge.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import collections.abc
import logging
import traceback

Expand All @@ -9,29 +8,9 @@
import cnudie.util
import oci.client as oc
import oci.model as om
import version

logger = logging.getLogger(__name__)


def iter_componentversions_to_purge(
component: ocm.Component | ocm.ComponentDescriptor,
policy: version.VersionRetentionPolicies,
oci_client: oc.Client,
) -> collections.abc.Generator[ocm.ComponentIdentity, None, None]:
oci_ref = cnudie.util.oci_ref(component=component)
if isinstance(component, ocm.ComponentDescriptor):
component = component.component

for v in version.versions_to_purge(
versions=oci_client.tags(oci_ref.ref_without_tag),
reference_version=component.version,
policy=policy,
):
yield ocm.ComponentIdentity(
name=component.name,
version=v,
)
logger = logging.getLogger(__name__)


def remove_component_descriptor_and_referenced_artefacts(
Expand Down
97 changes: 5 additions & 92 deletions concourse/model/traits/component_descriptor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,9 @@

from ci.util import not_none
from ocm import Label
import ci.util
import cnudie.retrieve
import ocm
import model.base
import version

from concourse.model.job import (
JobVariant,
Expand Down Expand Up @@ -89,60 +87,14 @@ class UploadMode(enum.StrEnum):
AttributeSpec.optional(
name='retention_policy',
default=None,
type=typing.Union[version.VersionRetentionPolicies, str],
doc='''
specifies how (old) component-descriptors and their referenced resources should be
handled. This is foremostly intended as an option for automated cleanup for
components with frequent (shortlived) releases and/or frequent (shortlived) snapshots.
if no retention_policy is defined, no cleanup will be done.
retention policy may either be defined "inline" (as a mapping value) or by referencing
a pre-defined policy by name (see `retentions_policies` attribute). In the latter case,
use the policie's name as (string) attribute value.
''',
type=str,
doc='obsolete',
),
AttributeSpec.optional(
name='retention_policies',
default=[
version.VersionRetentionPolicies(
name='clean-snapshots',
rules=[
version.VersionRetentionPolicy(
name='clean-snapshots',
keep=64,
match=version.VersionType.SNAPSHOT,
),
version.VersionRetentionPolicy(
name='keep-releases',
keep='all',
match=version.VersionType.RELEASE,
),
],
dry_run=False,
),
version.VersionRetentionPolicies(
name='clean-snapshots-and-releases',
rules=[
version.VersionRetentionPolicy(
name='clean-snapshots',
keep=64,
match=version.VersionType.SNAPSHOT,
),
version.VersionRetentionPolicy(
name='clean-releases',
keep=128,
match=version.VersionType.RELEASE,
),
],
dry_run=False,
),
],
type=typing.List[version.VersionRetentionPolicies],
doc='''
predefined retention policies (see default value). may be referenced via
`retention_policy` attribute (adding additional policies here has no immediate effect)
'''
default=[],
type=typing.List[str],
doc='obsolete'
),
AttributeSpec.deprecated(
name='validation_policies',
Expand Down Expand Up @@ -229,45 +181,6 @@ def step_name(self):
def upload(self) -> UploadMode:
return UploadMode(self.raw['upload'])

def retention_policy(self, raw=True) -> version.VersionRetentionPolicies | None:
if not (policy := self.raw.get('retention_policy', None)):
return None

if isinstance(policy, str):
# lookup name
for candidate in self.raw.get('retention_policies', ()):
if isinstance(candidate, dict):
name = candidate['name']
elif isinstance(candidate, version.VersionRetentionPolicies):
name = candidate.name
else:
raise ValueError(candidate)

if name == policy:
policy = candidate
break
else:
raise ValueError(f'did not find {policy=} in retention_policies')

if raw:
if isinstance(policy, version.VersionRetentionPolicies):
policy = dataclasses.asdict(
policy,
dict_factory=ci.util.dict_factory_enum_serialisiation,
)
return policy

if isinstance(policy, version.VersionRetentionPolicies):
return policy

return dacite.from_dict(
data_class=version.VersionRetentionPolicies,
data=policy,
config=dacite.Config(
cast=(enum.Enum,),
),
)

def resolve_dependencies(self):
return self.raw['resolve_dependencies']

Expand Down
55 changes: 0 additions & 55 deletions concourse/steps/component_descriptor.mako
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ if descriptor_trait.ocm_repository:
ocm_repository_url = descriptor_trait.ocm_repository.oci_ref
else:
ocm_repository_url = None
retention_policy = descriptor_trait.retention_policy()
ocm_repository_mappings = descriptor_trait.ocm_repository_mappings()
# label main repo as main
Expand Down Expand Up @@ -333,60 +332,6 @@ else:
)
with open(dependencies_path) as f:
print(f.read())
% if retention_policy:
logger.info('will honour retention-policy')
retention_policy = dacite.from_dict(
data_class=version.VersionRetentionPolicies,
data=${retention_policy},
config=dacite.Config(cast=(enum.Enum,)),
)
pprint.pprint(retention_policy)
if retention_policy.dry_run:
logger.info('dry-run - will only print versions to remove, but not actually remove them')
else:
logger.info('!! will attempt to remove listed component-versions, according to policy')
logger.info('the following versions were identified for being purged')
component = descriptor_v2.component
for idx, component_id in enumerate(cnudie.purge.iter_componentversions_to_purge(
component=component,
policy=retention_policy,
oci_client=oci_client,
)):
if idx >= 64:
print('will abort the purge, considering there seem to be more than 64 versions to cleanup')
print('this is done to limit execution-time - the purge will continue on next execution')
exit(0)
print(f'{idx} {component_id.name}:{component_id.version}')
if retention_policy.dry_run:
continue
component_to_purge = component_descriptor_lookup(
ocm.ComponentIdentity(
name=component.name,
version=component_id.version,
)
)
if not component_to_purge:
logger.warning(f'{component.name}:{component_id.version} was not found - ignore')
continue
try:
cnudie.purge.remove_component_descriptor_and_referenced_artefacts(
component=component_to_purge,
oci_client=oci_client,
lookup=component_descriptor_lookup,
recursive=False,
)
except Exception as e:
logger.warning(f'error occurred while trying to purge {component_id}: {e}')
traceback.print_exc()
% else:
logger.info('no retention-policy was defined - will not purge component-descriptors')
% endif
</%def>

<%def
Expand Down
14 changes: 0 additions & 14 deletions doc/traits/component_descriptor.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,20 +205,6 @@ Gardener-Components have a name that is by convention the github-repo-url (w/o s
- name: imagevector.gardener.cloud/source-repository
value: github.com/gardener/gardener # github-repo
Retention Policies (aka cleaning up old versions)
=================================================

The `retention_policies` attribute can be used to configure automated removal of
component descriptors and referenced `resources` (mostly OCI Container Images).

.. attention::
Removal of component descriptors and referenced resources is _permanent_. There is no
backup mechanism in place. Use with care. For example, if multiple component descriptors
share reference to the same OCI Artefact (using the same registry, repository, and tag)
removal of any of the referencing component descriptors will lead to stale references
in other component descriptors.

Cleanup Semantics and Use-Case
------------------------------

Expand Down
82 changes: 0 additions & 82 deletions version.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@


import collections
import dataclasses
import enum
import logging
import semver
Expand Down Expand Up @@ -41,47 +40,6 @@ class VersionType(enum.Enum):
ANY = 'any'


@dataclasses.dataclass(frozen=True)
class VersionRetentionPolicy:
name: str = None
keep: typing.Union[str, int] = 'all'
match: VersionType = VersionType.ANY
restrict: VersionRestriction = VersionRestriction.NONE
recursive: bool = False

def matches_version_restriction(self, version, ref_version) -> bool:
version = parse_to_semver(version)
final = is_final(version)

if self.match is VersionType.ANY:
pass
if self.match is VersionType.SNAPSHOT and final:
return False
if self.match is VersionType.RELEASE and not final:
return False

# if this line is reached, version-type matches

if self.restrict is VersionRestriction.NONE:
return True
elif self.restrict is VersionRestriction.SAME_MINOR:
ref_version = version.parse_to_semver(ref_version)
return ref_version.minor == version.minor
else:
raise RuntimeError(f'not implemented: {self.restrict}')

@property
def keep_all(self) -> bool:
return self.keep == 'all'


@dataclasses.dataclass
class VersionRetentionPolicies:
name: str
rules: list[VersionRetentionPolicy]
dry_run: bool = True


T = typing.TypeVar('T')


Expand All @@ -95,46 +53,6 @@ def is_final(
return not version.build and not version.prerelease


def versions_to_purge(
versions: typing.Iterable[T],
reference_version: Version,
policy: VersionRetentionPolicies,
converter: typing.Callable[[T], Version]=None,
) -> typing.Generator[T, None, None]:
versions_by_policy = collections.defaultdict(list)

def to_version(v: T):
if converter:
v = converter(v)
return v

for v in versions:
converted_version = to_version(v)
for rule in policy.rules:
rule: VersionRetentionPolicy
if rule.matches_version_restriction(
version=converted_version,
ref_version=reference_version,
):
versions_by_policy[rule].append(v)
break # first rule matches
else:
continue
else:
logger.info(f'no rule matched {converted_version}')

for policy, versions in versions_by_policy.items():
policy: VersionRetentionPolicy
if policy.keep_all:
continue

yield from smallest_versions(
versions=versions,
keep=policy.keep,
converter=converter,
)


def parse_to_semver(
version,
invalid_semver_ok: bool=False,
Expand Down

0 comments on commit 8377c7e

Please sign in to comment.