Introduce dedicated cfg-mgmt python distribution package

Especially hyperscaler SDKs are heavyweight, however we only use those
in very few occasions. With this python distribution package we will be
able to install them only if required, and remove from general purpose
"job image".

.ci/write-version

@@ -9,6 +9,7 @@ versionfiles=(
   oci/VERSION
   ocm/VERSION
   whd/VERSION
+  cfg_mgmt/VERSION
   cli/gardener_ci/VERSION
   concourse/resources/LAST_RELEASED_TAG
 )

.github/workflows/build-and-test.yaml

@@ -163,6 +163,7 @@ jobs:
           setup.oci.py \
           setup.ocm.py \
           setup.whd.py \
+          setup.cfg_mgmt.py \
         ; do
          echo "building distribution package from ${path}"
          python3 ${path} \
@@ -190,6 +191,7 @@ jobs:
           gardener-cicd-whd \
           gardener-cicd-cli \
           gardener-cicd-libs \
+          gardener-cicd-cfg-mgmt \
           gardener-gha-libs \
           gardener-oci \
           gardener-ocm \
@@ -396,6 +398,7 @@ jobs:
           --find-links /tmp/dist \
           gardener-cicd-libs \
           gardener-cicd-cli \
+          gardener-cicd-cfg-mgmt \
           gardener-oci \
           gardener-ocm \
           bandit \
@@ -449,6 +452,7 @@ jobs:
           --find-links /tmp/dist \
           gardener-cicd-libs \
           gardener-cicd-cli \
+          gardener-cicd-cfg-mgmt \
           gardener-oci \
           gardener-ocm \
           pytest \

.github/workflows/release.yaml

@@ -6,6 +6,7 @@ on:
         required: false
         type: boolean
         default: false
+        description: Whether to release to PYPI
 
 defaults:
   run:

cfg_mgmt/VERSION

@@ -0,0 +1 @@
+1.2602.0-dev

cli/gardener_ci/_cfg_mgmt.py

@@ -1,11 +1,15 @@
 import logging
 import re
 
-import cfg_mgmt.gcp
-import cfg_mgmt.reporting as cmr
-import cfg_mgmt.rotate
-import cfg_mgmt.model
-import cfg_mgmt.util as cmu
+try:
+    import cfg_mgmt.gcp
+    import cfg_mgmt.reporting as cmr
+    import cfg_mgmt.rotate
+    import cfg_mgmt.model
+    import cfg_mgmt.util as cmu
+except:
+    print('unable to import cfg_mgmt package, have you tried installing "gardener-cicd-cfg-mgmt"?')
+    raise
 import model
 
 

concourse/steps/cfg_reporting.mako

@@ -17,7 +17,11 @@ import dacite
 import ccc.delivery
 import ccc.github
 import concourse.util
-import cfg_mgmt.reporting as cmr
+try:
+  import cfg_mgmt.reporting as cmr
+except:
+  print('unable to import cfg_mgmt package, have you tried installing "gardener-cicd-cfg-mgmt"?')
+  raise
 import ci.util
 import github.compliance.model as gcm
 import github.compliance.report as gcr

concourse/steps/cfg_reporting.py

@@ -1,5 +1,8 @@
-
-import cfg_mgmt.model as cmm
+try:
+    import cfg_mgmt.model as cmm
+except:
+    print('unable to import cfg_mgmt package, have you tried installing "gardener-cicd-cfg-mgmt"?')
+    raise
 import github.compliance.issue as gci
 import github.compliance.model as gcm
 

concourse/steps/replicate_secrets.mako

@@ -20,7 +20,11 @@ ${step_lib('replicate_secrets')}
 
 import ccc.elasticsearch
 import ccc.github
-import cfg_mgmt.reporting as cmr
+try:
+  import cfg_mgmt.reporting as cmr
+except:
+  print('unable to import cfg_mgmt package, have you tried installing "gardener-cicd-cfg-mgmt"?')
+  raise
 import model
 import model.concourse
 import model.config_repo

concourse/steps/replicate_secrets.py

@@ -3,15 +3,23 @@
 import random
 import urllib3.exceptions
 
-import kubernetes.client
-import kubernetes.config
+try:
+    import kubernetes.client
+    import kubernetes.config
+except:
+    print('unable to import kubernetes package, have you tried installing "gardener-cicd-cfg-mgmt"?')
+    raise
 
 import ccc.delivery
 import ccc.github
 import ccc.secrets_server
-import cfg_mgmt.model as cmm
-import cfg_mgmt.reporting as cmr
-import cfg_mgmt.util as cmu
+try:
+    import cfg_mgmt.model as cmm
+    import cfg_mgmt.reporting as cmr
+    import cfg_mgmt.util as cmu
+except:
+    print('unable to import cfg_mgmt package, have you tried installing "gardener-cicd-cfg-mgmt"?')
+    raise
 import ci.log
 import ci.util
 import model

requirements-dev.txt

@@ -3,3 +3,4 @@
 -r requirements.ocm.txt
 -r requirements.txt
 -r requirements.whd.txt
+-r requirements.cfg_mgmt.txt

requirements.cfg_mgmt.txt

@@ -0,0 +1,13 @@
+aliyun-python-sdk-core
+aliyun-python-sdk-ecs
+aliyun-python-sdk-ram
+boto3
+google-api-core<3
+google-api-python-client<3
+google-auth
+google-cloud-storage<3
+google-crc32c>1.1.4
+kubernetes>=26.0.0
+msal<2
+openstacksdk
+oss2

requirements.txt

@@ -2,12 +2,8 @@
 GitPython
 Mako<2.0.0
 Sphinx
-aliyun-python-sdk-core
-aliyun-python-sdk-ecs
-aliyun-python-sdk-ram
 awesomeversion
 bcrypt<5.0.0
-boto3
 cachecontrol<1
 cachetools
 cryptography
@@ -21,18 +17,9 @@ ensure
 flake8
 gardener-oci
 github3.py<5.0.0
-google-api-core<3
-google-api-python-client<3
-google-auth
-google-cloud-storage<3
-google-crc32c>1.1.4
 html2text
 jsonschema
 jsonpath-ng
-kubernetes>=26.0.0
-msal<2
-openstacksdk
-oss2
 pycryptodome
 pyjwt
 pylama

setup.cfg_mgmt.py

@@ -0,0 +1,34 @@
+import setuptools
+import os
+
+own_dir = os.path.abspath(os.path.dirname(__file__))
+
+
+def requirements():
+    yield 'gardener-cicd-libs'
+
+    with open(os.path.join(own_dir, 'requirements.cfg_mgmt.txt')) as f:
+        for line in f.readlines():
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+
+            yield line
+
+
+def version():
+    with open(os.path.join(own_dir, 'cfg_mgmt', 'VERSION')) as f:
+        return f.read().strip()
+
+
+setuptools.setup(
+    name='gardener-cicd-cfg-mgmt',
+    version=version(),
+    description='Gardener CI/CD Config Management',
+    long_description='Gardener CI/CD Config Management',
+    long_description_content_type='text/markdown',
+    python_requires='>=3.12',
+    packages=['cfg_mgmt'],
+    install_requires=list(requirements()),
+    entry_points={},
+)

setup.gha.py

@@ -107,6 +107,7 @@ def modules():
     module_names.remove('setup.oci')
     module_names.remove('setup.ocm')
     module_names.remove('setup.whd')
+    module_names.remove('setup.cfg_mgmt')
     return module_names
 
 

setup.py

@@ -29,6 +29,8 @@ def modules():
     module_names.remove('setup.oci')
     module_names.remove('setup.ocm')
     module_names.remove('setup.whd')
+    module_names.remove('setup.cfg_mgmt')
+
     return module_names
 
 
@@ -39,6 +41,7 @@ def packages():
     package_names.remove('whd')
     package_names.remove('oci')
     package_names.remove('ocm')
+    package_names.remove('cfg_mgmt')
     return package_names