Fix IT tests after version bumps (#15827) (#15839)

This commit fixes IT failures that frequently occur after
version bumps due to missing unified release snapshot builds for
the new version.

This commit uses project specific DRA snapshot URLs for ES and Filebeat
in all cases apart from release builds.

(cherry picked from commit d74fea4)

build.gradle

@@ -41,6 +41,7 @@ import org.yaml.snakeyaml.Yaml
 import de.undercouch.gradle.tasks.download.Download
 import groovy.json.JsonSlurper
 import org.logstash.gradle.tooling.ListProjectDependencies
+import org.logstash.gradle.tooling.SnapshotArtifactURLs
 
 allprojects {
  group = 'org.logstash'
@@ -162,12 +163,17 @@ tasks.register("configureArtifactInfo") {
  version = "$version-$versionQualifier"
  }
 
- def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
+ boolean isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
  String apiResponse = artifactVersionsApi.toURL().text
 
  def dlVersions = new JsonSlurper().parseText(apiResponse)
  String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
  if (qualifiedVersion == null) {
+ if (!isReleaseBuild) {
+ project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
+ project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
+ return
+ }
  throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
  }
  // find latest reference to last build
@@ -178,6 +184,7 @@ tasks.register("configureArtifactInfo") {
 
  project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
  project.ext.set("stackArtifactSuffix", qualifiedVersion)
+ project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
  }
 }
 
@@ -478,16 +485,23 @@ tasks.register("downloadFilebeat") {
 
  doLast {
  download {
- String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}"
+ String beatVersion = project.ext.get("stackArtifactSuffix")
+ String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
  project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
 
- // find url of build artifact
- String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
- String apiResponse = artifactApiUrl.toURL().text
- def buildUrls = new JsonSlurper().parseText(apiResponse)
-
- project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
- project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
+ if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
+ def res = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
+ project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
+ project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
+ } else {
+ // find url of build artifact
+ String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
+ String apiResponse = artifactApiUrl.toURL().text
+ def buildUrls = new JsonSlurper().parseText(apiResponse)
+
+ project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
+ project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
+ }
 
  src project.ext.filebeatSnapshotUrl
  onlyIfNewer true
@@ -523,14 +537,20 @@ tasks.register("checkEsSHA") {
  description "Download ES version remote's fingerprint file"
 
  doLast {
- String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
-
-
- // find url of build artifact
- String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
- String apiResponse = artifactApiUrl.toURL().text
- def buildUrls = new JsonSlurper().parseText(apiResponse)
- String remoteSHA = buildUrls.package.sha_url.toURL().text
+ String esVersion = project.ext.get("stackArtifactSuffix")
+ String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
+ String remoteSHA
+
+ if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
+ def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
+ remoteSHA = res.packageShaUrl
+ } else {
+ // find url of build artifact
+ String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
+ String apiResponse = artifactApiUrl.toURL().text
+ def buildUrls = new JsonSlurper().parseText(apiResponse)
+ remoteSHA = buildUrls.package.sha_url.toURL().text
+ }
 
  def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
  if (localESArchive.exists()) {
@@ -564,16 +584,24 @@ tasks.register("downloadEs") {
 
  doLast {
  download {
- String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
- project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}")
-
- // find url of build artifact
- String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
- String apiResponse = artifactApiUrl.toURL().text
- def buildUrls = new JsonSlurper().parseText(apiResponse)
-
- project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
- project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+ String esVersion = project.ext.get("stackArtifactSuffix")
+ String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
+
+ project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
+
+ if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
+ def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
+ project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
+ project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+ } else {
+ // find url of build artifact
+ String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
+ String apiResponse = artifactApiUrl.toURL().text
+ def buildUrls = new JsonSlurper().parseText(apiResponse)
+
+ project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
+ project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+ }
 
  src project.ext.elasticsearchSnapshotURL
  onlyIfNewer true

buildSrc/src/main/groovy/org/logstash/gradle/tooling/SnapshotArtifactURLs.groovy

@@ -0,0 +1,28 @@
+package org.logstash.gradle.tooling
+
+import groovy.json.JsonSlurper
+
+/**
+ * Helper class to obtain project specific (e.g. Elasticsearch or beats/filebeat) snapshot-DRA artifacts.
+ * We use it in all cases apart from release builds.
+ * */
+class SnapshotArtifactURLs {
+ /**
+ * Returns a list of the package and package SHA(512) URLs for a given project / version / downloadedPackageName
+ * */
+ static def packageUrls(String project, String projectVersion, String downloadedPackageName) {
+ String artifactSnapshotVersionsApiPrefix = "https://artifacts-snapshot.elastic.co"
+
+ // e.g. https://artifacts-snapshot.elastic.co/elasticsearch/latest/8.11.5-SNAPSHOT.json
+ String apiResponse = "${artifactSnapshotVersionsApiPrefix}/${project}/latest/${projectVersion}.json".toURL().text
+ def artifactUrls = new JsonSlurper().parseText(apiResponse)
+ String manifestUrl = artifactUrls["manifest_url"]
+ // e.g. https://artifacts-snapshot.elastic.co/elasticsearch/8.11.5-12345678/manifest-8.11.5-SNAPSHOT.json
+ apiResponse = manifestUrl.toURL().text
+ def packageArtifactUrls = new JsonSlurper().parseText(apiResponse)
+ String packageUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["url"]
+ String packageShaUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["sha_url"]
+
+ return ["packageUrl": packageUrl, "packageShaUrl": packageShaUrl]
+ }
+}