From b3b31b2d10f6a9266e70acce0a02e0f6cc56478b Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 8 May 2024 21:39:54 +0200 Subject: [PATCH 1/4] Use the yum staging SSH key as a Jenkins secret Rather than relying on Puppet to deploy an SSH key, this stores a key in Jenkins and then uses an SSH-agent to provide access. --- theforeman.org/pipelines/lib/packaging.groovy | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/theforeman.org/pipelines/lib/packaging.groovy b/theforeman.org/pipelines/lib/packaging.groovy index 3ea691e0..3d9e6804 100644 --- a/theforeman.org/pipelines/lib/packaging.groovy +++ b/theforeman.org/pipelines/lib/packaging.groovy @@ -482,16 +482,15 @@ def rsync_debian(user, ssh_key, suite, component, deb_paths) { } def rsync_to_yum_stage(collection, version) { - def ssh_key = '/home/jenkins/workspace/staging_key/rsync_yumrepostage_key' - if (!fileExists('upload_stage_rpms')) { git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false } - sh """ - export RSYNC_RSH="ssh -i ${ssh_key}" - export VERSION=${version} - export PROJECT=${collection} - ./upload_stage_rpms - """ + sshagent(['yum-stage']) { + sh """ + export VERSION=${version} + export PROJECT=${collection} + ./upload_stage_rpms + """ + } } From b9b501ec84890d7d9a7930938d4b809af1658d6a Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 8 May 2024 22:07:51 +0200 Subject: [PATCH 2/4] Align all pipelines on generate_stage_repository script This creates a single stage to both build and push staging RPMs by using the generate_stage_repository script from theforeman-rel-eng. It uses environment variables to configure things, which further removes unique configuration from individual pipeline steps. --- theforeman.org/pipelines/lib/packaging.groovy | 14 ++++------- .../release/pipelines/candlepin.groovy | 25 ++++++------------- .../pipelines/release/pipelines/client.groovy | 25 ++++++------------- .../release/pipelines/foreman-rpm.groovy | 25 ++++++------------- .../release/pipelines/katello.groovy | 25 ++++++------------- .../release/pipelines/plugins.groovy | 20 ++++++--------- .../release/pipelines/pulpcore.groovy | 25 ++++++------------- 7 files changed, 52 insertions(+), 107 deletions(-) diff --git a/theforeman.org/pipelines/lib/packaging.groovy b/theforeman.org/pipelines/lib/packaging.groovy index 3d9e6804..3f15d746 100644 --- a/theforeman.org/pipelines/lib/packaging.groovy +++ b/theforeman.org/pipelines/lib/packaging.groovy @@ -481,16 +481,12 @@ def rsync_debian(user, ssh_key, suite, component, deb_paths) { } } -def rsync_to_yum_stage(collection, version) { - if (!fileExists('upload_stage_rpms')) { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false - } +def rsync_to_yum_stage() { + git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + + sh "./generate_stage_repository" sshagent(['yum-stage']) { - sh """ - export VERSION=${version} - export PROJECT=${collection} - ./upload_stage_rpms - """ + sh "./upload_stage_rpms" } } diff --git a/theforeman.org/pipelines/release/pipelines/candlepin.groovy b/theforeman.org/pipelines/release/pipelines/candlepin.groovy index 730a1b58..6c592c99 100644 --- a/theforeman.org/pipelines/release/pipelines/candlepin.groovy +++ b/theforeman.org/pipelines/release/pipelines/candlepin.groovy @@ -8,28 +8,19 @@ pipeline { ansiColor('xterm') } - stages { - stage('staging-build-repository') { - when { - expression { candlepin_version == 'nightly' } - } - steps { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + environment { + PROJECT = 'candlepin' + VERSION = candlepin_version + } - script { - candlepin_distros.each { distro -> - sh "./build_stage_repository candlepin ${candlepin_version} ${distro}" - } - } - } - } - stage('staging-copy-repository') { + stages { + stage('staging-repository') { when { - expression { candlepin_version == 'nightly' } + expression { env.VERSION == 'nightly' } } steps { script { - rsync_to_yum_stage('candlepin', candlepin_version) + rsync_to_yum_stage } } } diff --git a/theforeman.org/pipelines/release/pipelines/client.groovy b/theforeman.org/pipelines/release/pipelines/client.groovy index 5d5d9e3d..e03dbbc1 100644 --- a/theforeman.org/pipelines/release/pipelines/client.groovy +++ b/theforeman.org/pipelines/release/pipelines/client.groovy @@ -8,28 +8,19 @@ pipeline { ansiColor('xterm') } - stages { - stage('staging-build-repository') { - when { - expression { foreman_version == 'nightly' } - } - steps { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + environment { + PROJECT = 'client' + VERSION = foreman_version + } - script { - foreman_client_distros.each { distro -> - sh "./build_stage_repository client ${foreman_version} ${distro}" - } - } - } - } - stage('staging-copy-repository') { + stages { + stage('staging-repository') { when { - expression { foreman_version == 'nightly' } + expression { env.VERSION == 'nightly' } } steps { script { - rsync_to_yum_stage('client', foreman_version) + rsync_to_yum_stage } } } diff --git a/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy b/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy index aa87cc38..aa76df0d 100644 --- a/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy +++ b/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy @@ -8,28 +8,19 @@ pipeline { ansiColor('xterm') } - stages { - stage('staging-build-repository') { - when { - expression { foreman_version == 'nightly' } - } - steps { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + environment { + PROJECT = 'foreman' + VERSION = foreman_version + } - script { - foreman_el_releases.each { distro -> - sh "./build_stage_repository foreman ${foreman_version} ${distro}" - } - } - } - } - stage('staging-copy-repository') { + stages { + stage('staging-repository') { when { - expression { foreman_version == 'nightly' } + expression { env.VERSION == 'nightly' } } steps { script { - rsync_to_yum_stage('foreman', foreman_version) + rsync_to_yum_stage } } } diff --git a/theforeman.org/pipelines/release/pipelines/katello.groovy b/theforeman.org/pipelines/release/pipelines/katello.groovy index 597b155e..0545cb34 100644 --- a/theforeman.org/pipelines/release/pipelines/katello.groovy +++ b/theforeman.org/pipelines/release/pipelines/katello.groovy @@ -8,28 +8,19 @@ pipeline { ansiColor('xterm') } - stages { - stage('staging-build-repository') { - when { - expression { katello_version == 'nightly' } - } - steps { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + environment { + PROJECT = 'katello' + VERSION = katello_version + } - script { - foreman_el_releases.each { distro -> - sh "./build_stage_repository katello ${katello_version} ${distro} ${foreman_version}" - } - } - } - } - stage('staging-copy-repository') { + stages { + stage('staging-repository') { when { - expression { katello_version == 'nightly' } + expression { env.VERSION == 'nightly' } } steps { script { - rsync_to_yum_stage('katello', katello_version) + rsync_to_yum_stage } } } diff --git a/theforeman.org/pipelines/release/pipelines/plugins.groovy b/theforeman.org/pipelines/release/pipelines/plugins.groovy index eec3099a..11cf06a7 100644 --- a/theforeman.org/pipelines/release/pipelines/plugins.groovy +++ b/theforeman.org/pipelines/release/pipelines/plugins.groovy @@ -8,22 +8,16 @@ pipeline { ansiColor('xterm') } - stages { - stage('staging-build-repository') { - steps { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + environment { + PROJECT = 'plugins' + VERSION = foreman_version + } - script { - foreman_el_releases.each { distro -> - sh "./build_stage_repository plugins ${foreman_version} ${distro}" - } - } - } - } - stage('staging-copy-repository') { + stages { + stage('staging-repository') { steps { script { - rsync_to_yum_stage('plugins', foreman_version) + rsync_to_yum_stage } } } diff --git a/theforeman.org/pipelines/release/pipelines/pulpcore.groovy b/theforeman.org/pipelines/release/pipelines/pulpcore.groovy index 0112c58e..9eae7f81 100644 --- a/theforeman.org/pipelines/release/pipelines/pulpcore.groovy +++ b/theforeman.org/pipelines/release/pipelines/pulpcore.groovy @@ -8,28 +8,19 @@ pipeline { ansiColor('xterm') } - stages { - stage('staging-build-repository') { - when { - expression { pulpcore_version == 'nightly' } - } - steps { - git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false + environment { + PROJECT = 'pulpcore' + VERSION = pulpcore_version + } - script { - pulpcore_distros.each { distro -> - sh "./build_stage_repository pulpcore ${pulpcore_version} ${distro}" - } - } - } - } - stage('staging-copy-repository') { + stages { + stage('staging-repository') { when { - expression { pulpcore_version == 'nightly' } + expression { env.VERSION == 'nightly' } } steps { script { - rsync_to_yum_stage('pulpcore', pulpcore_version) + rsync_to_yum_stage } } } From 9892fd55de8001826056df53446a1211aa32e138 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Tue, 4 Jun 2024 17:56:35 +0200 Subject: [PATCH 3/4] fixup! Align all pipelines on generate_stage_repository script --- theforeman.org/pipelines/release/pipelines/candlepin.groovy | 5 ++++- theforeman.org/pipelines/release/pipelines/client.groovy | 5 ++++- .../pipelines/release/pipelines/foreman-rpm.groovy | 5 ++++- theforeman.org/pipelines/release/pipelines/katello.groovy | 5 ++++- theforeman.org/pipelines/release/pipelines/plugins.groovy | 5 ++++- theforeman.org/pipelines/release/pipelines/pulpcore.groovy | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/theforeman.org/pipelines/release/pipelines/candlepin.groovy b/theforeman.org/pipelines/release/pipelines/candlepin.groovy index 6c592c99..63cdc9ac 100644 --- a/theforeman.org/pipelines/release/pipelines/candlepin.groovy +++ b/theforeman.org/pipelines/release/pipelines/candlepin.groovy @@ -10,7 +10,10 @@ pipeline { environment { PROJECT = 'candlepin' - VERSION = candlepin_version + } + + script { + env.VERSION = candlepin_version } stages { diff --git a/theforeman.org/pipelines/release/pipelines/client.groovy b/theforeman.org/pipelines/release/pipelines/client.groovy index e03dbbc1..eef7e5bb 100644 --- a/theforeman.org/pipelines/release/pipelines/client.groovy +++ b/theforeman.org/pipelines/release/pipelines/client.groovy @@ -10,7 +10,10 @@ pipeline { environment { PROJECT = 'client' - VERSION = foreman_version + } + + script { + env.VERSION = foreman_version } stages { diff --git a/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy b/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy index aa76df0d..bfd695ad 100644 --- a/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy +++ b/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy @@ -10,7 +10,10 @@ pipeline { environment { PROJECT = 'foreman' - VERSION = foreman_version + } + + script { + env.VERSION = foreman_version } stages { diff --git a/theforeman.org/pipelines/release/pipelines/katello.groovy b/theforeman.org/pipelines/release/pipelines/katello.groovy index 0545cb34..abc589df 100644 --- a/theforeman.org/pipelines/release/pipelines/katello.groovy +++ b/theforeman.org/pipelines/release/pipelines/katello.groovy @@ -10,7 +10,10 @@ pipeline { environment { PROJECT = 'katello' - VERSION = katello_version + } + + script { + env.VERSION = katello_version } stages { diff --git a/theforeman.org/pipelines/release/pipelines/plugins.groovy b/theforeman.org/pipelines/release/pipelines/plugins.groovy index 11cf06a7..3704fb10 100644 --- a/theforeman.org/pipelines/release/pipelines/plugins.groovy +++ b/theforeman.org/pipelines/release/pipelines/plugins.groovy @@ -10,7 +10,10 @@ pipeline { environment { PROJECT = 'plugins' - VERSION = foreman_version + } + + script { + env.VERSION = foreman_version } stages { diff --git a/theforeman.org/pipelines/release/pipelines/pulpcore.groovy b/theforeman.org/pipelines/release/pipelines/pulpcore.groovy index 9eae7f81..7a169070 100644 --- a/theforeman.org/pipelines/release/pipelines/pulpcore.groovy +++ b/theforeman.org/pipelines/release/pipelines/pulpcore.groovy @@ -10,7 +10,10 @@ pipeline { environment { PROJECT = 'pulpcore' - VERSION = pulpcore_version + } + + script { + env.VERSION = pulpcore_version } stages { From 05e396253255c18f96bc97862c518c0866a81914 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 8 May 2024 22:16:38 +0200 Subject: [PATCH 4/4] Further use environment variables to align RPM pipelines This unifies RPM pipelines further by using environment variables that are already available. It makes some changes to Discourse to align them, other than that there should be no practical difference here. --- .../pipelines/release/pipelines/candlepin.groovy | 8 ++++---- theforeman.org/pipelines/release/pipelines/client.groovy | 4 ++-- .../pipelines/release/pipelines/foreman-rpm.groovy | 8 ++++---- theforeman.org/pipelines/release/pipelines/katello.groovy | 6 +++--- theforeman.org/pipelines/release/pipelines/plugins.groovy | 6 +++--- .../pipelines/release/pipelines/pulpcore.groovy | 8 ++++---- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/theforeman.org/pipelines/release/pipelines/candlepin.groovy b/theforeman.org/pipelines/release/pipelines/candlepin.groovy index 63cdc9ac..fdbc280b 100644 --- a/theforeman.org/pipelines/release/pipelines/candlepin.groovy +++ b/theforeman.org/pipelines/release/pipelines/candlepin.groovy @@ -30,7 +30,7 @@ pipeline { stage('staging-repoclosure') { steps { script { - parallel repoclosures('candlepin', candlepin_distros, candlepin_version) + parallel repoclosures(env.PROJECT, candlepin_distros, env.VERSION) } } post { @@ -44,7 +44,7 @@ pipeline { steps { script { - runDuffyPipeline('candlepin-rpm', candlepin_version) + runDuffyPipeline("${env.PROJECT}-rpm", env.VERSION) } } } @@ -54,7 +54,7 @@ pipeline { steps { script { candlepin_distros.each { distro -> - push_foreman_staging_rpms('candlepin', candlepin_version, distro) + push_foreman_staging_rpms(env.PROJECT, env.VERSION, distro) } } } @@ -62,7 +62,7 @@ pipeline { } post { failure { - notifyDiscourse(env, "Candlepin ${candlepin_version} RPM pipeline failed:", currentBuild.description) + notifyDiscourse(env, "${env.PROJECT} ${env.VERSION} RPM pipeline failed:", currentBuild.description) } } } diff --git a/theforeman.org/pipelines/release/pipelines/client.groovy b/theforeman.org/pipelines/release/pipelines/client.groovy index eef7e5bb..4b3a83a6 100644 --- a/theforeman.org/pipelines/release/pipelines/client.groovy +++ b/theforeman.org/pipelines/release/pipelines/client.groovy @@ -30,7 +30,7 @@ pipeline { stage('staging-repoclosure') { steps { script { - parallel repoclosures('foreman-client-staging', foreman_client_distros, foreman_version) + parallel repoclosures("foreman-${env.PROJECT}-staging", foreman_client_distros, env.VERSION) } } post { @@ -45,7 +45,7 @@ pipeline { steps { script { foreman_client_distros.each { distro -> - push_foreman_staging_rpms('client', foreman_version, distro) + push_foreman_staging_rpms(env.PROJECT, env.VERSION, distro) } } } diff --git a/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy b/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy index bfd695ad..588d5980 100644 --- a/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy +++ b/theforeman.org/pipelines/release/pipelines/foreman-rpm.groovy @@ -30,7 +30,7 @@ pipeline { stage('staging-repoclosure') { steps { script { - parallel repoclosures('foreman-staging', foreman_el_releases, foreman_version) + parallel repoclosures("${env.PROJECT}-staging", foreman_el_releases, env.VERSION) } } post { @@ -44,7 +44,7 @@ pipeline { steps { script { - runDuffyPipeline('foreman-rpm', foreman_version) + runDuffyPipeline("${env.PROJECT}-rpm", env.VERSION) } } } @@ -54,7 +54,7 @@ pipeline { steps { script { foreman_el_releases.each { distro -> - push_foreman_staging_rpms('foreman', foreman_version, distro) + push_foreman_staging_rpms(env.PROJECT, env.VERSION, distro) } } } @@ -62,7 +62,7 @@ pipeline { } post { failure { - notifyDiscourse(env, 'Foreman RPM nightly pipeline failed:', currentBuild.description) + notifyDiscourse(env, "${env.PROJECT} ${env.VERSION} RPM pipeline failed:", currentBuild.description) } } } diff --git a/theforeman.org/pipelines/release/pipelines/katello.groovy b/theforeman.org/pipelines/release/pipelines/katello.groovy index abc589df..a6d3004e 100644 --- a/theforeman.org/pipelines/release/pipelines/katello.groovy +++ b/theforeman.org/pipelines/release/pipelines/katello.groovy @@ -44,7 +44,7 @@ pipeline { steps { script { - runDuffyPipeline('katello-rpm', katello_version) + runDuffyPipeline("${env.PROJECT}-rpm", env.VERSION) } } } @@ -54,7 +54,7 @@ pipeline { steps { script { foreman_el_releases.each { distro -> - push_foreman_staging_rpms('katello', katello_version, distro) + push_foreman_staging_rpms(env.PROJECT, env.VERSION, distro) } } } @@ -62,7 +62,7 @@ pipeline { } post { failure { - notifyDiscourse(env, "Katello ${katello_version} pipeline failed:", currentBuild.description) + notifyDiscourse(env, "${env.PROJECT} ${env.VERSION} RPM pipeline failed:", currentBuild.description) } } } diff --git a/theforeman.org/pipelines/release/pipelines/plugins.groovy b/theforeman.org/pipelines/release/pipelines/plugins.groovy index 3704fb10..3c3359bf 100644 --- a/theforeman.org/pipelines/release/pipelines/plugins.groovy +++ b/theforeman.org/pipelines/release/pipelines/plugins.groovy @@ -27,7 +27,7 @@ pipeline { stage('staging-repoclosure') { steps { script { - parallel repoclosures('plugins-staging', foreman_el_releases, foreman_version) + parallel repoclosures("${env.PROJECT}-staging", foreman_el_releases, env.VERSION) } } post { @@ -42,7 +42,7 @@ pipeline { steps { script { foreman_el_releases.each { distro -> - push_foreman_staging_rpms('plugins', foreman_version, distro) + push_foreman_staging_rpms(env.PROJECT, env.VERSION, distro) } } } @@ -50,7 +50,7 @@ pipeline { } post { failure { - notifyDiscourse(env, "Plugins ${foreman_version} pipeline failed:", currentBuild.description) + notifyDiscourse(env, "${env.PROJECT} ${env.VERSION} RPM pipeline failed:", currentBuild.description) } } } diff --git a/theforeman.org/pipelines/release/pipelines/pulpcore.groovy b/theforeman.org/pipelines/release/pipelines/pulpcore.groovy index 7a169070..f787e864 100644 --- a/theforeman.org/pipelines/release/pipelines/pulpcore.groovy +++ b/theforeman.org/pipelines/release/pipelines/pulpcore.groovy @@ -30,7 +30,7 @@ pipeline { stage('staging-repoclosure') { steps { script { - parallel repoclosures('pulpcore-staging', foreman_el_releases, foreman_version) + parallel repoclosures("${env.PROJECT}-staging", pulpcore_distros, env.VERSION) } } post { @@ -44,7 +44,7 @@ pipeline { steps { script { - runDuffyPipeline('pulpcore-rpm', pulpcore_version) + runDuffyPipeline("${env.PROJECT}-rpm", env.VERSION) } } } @@ -54,7 +54,7 @@ pipeline { steps { script { pulpcore_distros.each { distro -> - push_foreman_staging_rpms('pulpcore', pulpcore_version, distro) + push_foreman_staging_rpms(env.PROJECT, env.VERSION, distro) } } } @@ -62,7 +62,7 @@ pipeline { } post { failure { - notifyDiscourse(env, "Pulpcore ${pulpcore_version} RPM pipeline failed:", currentBuild.description) + notifyDiscourse(env, "${env.PROJECT} ${env.VERSION} RPM pipeline failed:", currentBuild.description) } } }