From 5eafbf613038e0a5f221e4f60fbb4953c11b6a2a Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Tue, 5 Sep 2023 11:57:34 +0200 Subject: [PATCH 1/5] Increase timeout for Py 3.6 Medium-large and Py 3.9 Medium-large too high value to see if it finally finish before we dig into the tests (#15725) --- scripts/jenkins/groovy/defineTestStages.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/jenkins/groovy/defineTestStages.groovy b/scripts/jenkins/groovy/defineTestStages.groovy index 98c08a0042fe..a8786067b743 100644 --- a/scripts/jenkins/groovy/defineTestStages.groovy +++ b/scripts/jenkins/groovy/defineTestStages.groovy @@ -440,7 +440,7 @@ def call(final pipelineContext) { ], [ stageName: 'Py3.6 Medium-large', target: 'test-pyunit-medium-large', pythonVersion: '3.6', - timeoutValue: 220, component: pipelineContext.getBuildConfig().COMPONENT_PY + timeoutValue: 300, component: pipelineContext.getBuildConfig().COMPONENT_PY ], [ stageName: 'R3.3 Medium-large', target: 'test-r-medium-large', rVersion: '3.3.3', @@ -546,7 +546,7 @@ def call(final pipelineContext) { ], [ stageName: 'Py3.9 Medium-large', target: 'test-pyunit-medium-large', pythonVersion: '3.9', - timeoutValue: 170, component: pipelineContext.getBuildConfig().COMPONENT_PY + timeoutValue: 300, component: pipelineContext.getBuildConfig().COMPONENT_PY ], [ // These run with reduced number of file descriptors for early detection of FD leaks stageName: 'XGBoost Stress tests', target: 'test-pyunit-xgboost-stress', pythonVersion: '3.6', timeoutValue: 40, From 6dbe6ec8e1441a70cadfc156b7812c1ac846557d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Veronika=20Maurerov=C3=A1?= Date: Fri, 8 Sep 2023 16:50:17 +0200 Subject: [PATCH 2/5] Fix KMeans tests (#15720) --- h2o-algos/src/test/java/hex/kmeans/KMeansTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java b/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java index a476b0327ef4..6103516a88d3 100755 --- a/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java +++ b/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java @@ -37,16 +37,10 @@ private static KMeansModel doSeed( KMeansModel.KMeansParameters parms, long seed return kmm; } - //PUBDEV-871: Double-check the training metrics (gathered by computeStatsFillModel) and the scoring logic by scoring on the training set private static void checkConsistency(KMeansModel kmm) { - //FIXME: TODO: remove this false, and fix the algo! PUBDEV-871 - if (false) { KMeansModel.KMeansParameters parms = kmm._parms; Assert.assertTrue((ArrayUtils.sum(kmm._output._size) - parms.train().numRows()) <= 1); -// Log.info(kmm._output._model_summary); -// Log.info(kmm._output._scoring_history); -// Log.info(((ModelMetricsClustering)kmm._output._training_metrics).createCentroidStatsTable().toString()); kmm.score(parms.train()).delete(); //this scores on the training data and appends a ModelMetrics ModelMetricsClustering mm = (ModelMetricsClustering) ModelMetrics.getFromDKV(kmm, parms.train()); Assert.assertTrue(Arrays.equals(mm._size, ((ModelMetricsClustering) kmm._output._training_metrics)._size)); @@ -56,7 +50,6 @@ private static void checkConsistency(KMeansModel kmm) { Assert.assertTrue(MathUtils.compare(mm._totss, ((ModelMetricsClustering) kmm._output._training_metrics)._totss, 1e-6, 1e-6)); Assert.assertTrue(MathUtils.compare(mm._betweenss, ((ModelMetricsClustering) kmm._output._training_metrics)._betweenss, 1e-6, 1e-6)); Assert.assertTrue(MathUtils.compare(mm._tot_withinss, ((ModelMetricsClustering) kmm._output._training_metrics)._tot_withinss, 1e-6, 1e-6)); - } } @Test public void testIris() { From cb00db30463f301dfea66a56d98e1895db635d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Novotn=C3=BD?= Date: Tue, 12 Sep 2023 10:22:36 +0200 Subject: [PATCH 3/5] [GH-15736] Run Prisma Scan Pipeline also on Main Standalone Jar (#15740) * [GH-15736] Run Prisma Scan Pipeline also on Main Standalone Jar * Add return statement * Add composite stage * Remove return statement * Add steps * Remove steps from inner stages * Update titles --- docker/prisma/Dockerfile.mainjars | 2 + ...file.scanningjars => Dockerfile.steamjars} | 0 .../jenkinsfiles/Jenkinsfile-PrismaScan | 81 ++++++++++--------- 3 files changed, 43 insertions(+), 40 deletions(-) create mode 100644 docker/prisma/Dockerfile.mainjars rename docker/prisma/{Dockerfile.scanningjars => Dockerfile.steamjars} (100%) diff --git a/docker/prisma/Dockerfile.mainjars b/docker/prisma/Dockerfile.mainjars new file mode 100644 index 000000000000..c9eff19411b5 --- /dev/null +++ b/docker/prisma/Dockerfile.mainjars @@ -0,0 +1,2 @@ +FROM alpine:latest +COPY ./h2o-assemblies/main/build/libs/*.jar /tmp/ diff --git a/docker/prisma/Dockerfile.scanningjars b/docker/prisma/Dockerfile.steamjars similarity index 100% rename from docker/prisma/Dockerfile.scanningjars rename to docker/prisma/Dockerfile.steamjars diff --git a/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan b/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan index fd5eebaa8161..2a58ae9c62a3 100644 --- a/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan +++ b/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan @@ -3,15 +3,42 @@ @Library('test-shared-library') _ def dockerImage -def branchOrTag -def steamImage -pipeline { - agent { node { label 'linux&&docker' } } +def setScanningStages(assemblyType, stageIndex) { + def assemblyImage + stage("${stageIndex}.A. Scan ${assemblyType} jar using Prisma") { + script { + branchName = "${env.BRANCH_NAME}".replace('/', '-') + assemblyImage = "h2o-assemblies/${assemblyType}:${BUILD_NUMBER}-${branchName}" + + sh "docker build . -t ${assemblyImage} -f ./docker/prisma/Dockerfile.${assemblyType}jars" - parameters { - string(name: 'BRANCH_OR_TAG', defaultValue: 'master', description: 'Enter branch or tag you want to scan.') + // scan the image + prismaCloudScanImage ca: '', + cert: '', + dockerAddress: 'unix:///var/run/docker.sock', + image: "${assemblyImage}", + key: '', + logLevel: 'info', + podmanPath: '', + project: '', + resultsFile: "prisma-${assemblyType}-scan-results.json", + ignoreImageBuildTime: true + } + } + stage("${stageIndex}.B. Export results for ${assemblyType} jar to CSV") { + withCredentials([usernamePassword(credentialsId: 'twistlock_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "curl -k -u \$USERNAME:\$PASSWORD https://mr-0xz1:8083/api/v1/scans/download?search=${assemblyImage} > ${assemblyImage}.csv" + } + archiveArtifacts artifacts: "${assemblyImage}.csv" } + stage("${stageIndex}.C. Publish report for ${assemblyType} jar") { + prismaCloudPublish resultsFilePattern: "prisma-${assemblyType}-scan-results.json" + } +} + +pipeline { + agent { node { label 'linux&&docker' } } options { ansiColor('xterm') @@ -29,54 +56,28 @@ pipeline { } } - stage('1. Build jar') { + stage('1. Build jars') { steps { script{ dockerImage.inside(){ sh "./gradlew :h2o-assemblies:steam:shadowJar" + sh "./gradlew :h2o-assemblies:main:shadowJar" archiveArtifacts artifacts: "h2o-assemblies/steam/build/libs/*.jar" + archiveArtifacts artifacts: "h2o-assemblies/main/build/libs/*.jar" } } } } - stage('2. Scan jar using Prisma'){ + stage('2. Steam assembly jar') { steps { - script{ - branchOrTag = "${BRANCH_OR_TAG}".replace('/','-') - steamImage = "h2o-assemblies/steam:${BUILD_NUMBER}-${branchOrTag}" - - sh "docker build . -t ${steamImage} -f ./docker/prisma/Dockerfile.scanningjars" - - // scan the image - prismaCloudScanImage ca: '', - cert: '', - dockerAddress: 'unix:///var/run/docker.sock', - image: "${steamImage}", - key: '', - logLevel: 'info', - podmanPath: '', - project: '', - resultsFile: 'prisma-cloud-scan-results.json', - ignoreImageBuildTime:true - } - + setScanningStages("steam", 2) } } - stage('3. Export results to CSV'){ - steps{ - withCredentials([usernamePassword(credentialsId: 'twistlock_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "curl -k -u \$USERNAME:\$PASSWORD https://mr-0xz1:8083/api/v1/scans/download?search=${steamImage} > ${steamImage}.csv" - } - archiveArtifacts artifacts: "${steamImage}.csv" - } - } - stage('4. Publish report'){ - steps{ - prismaCloudPublish resultsFilePattern: 'prisma-cloud-scan-results.json' + stage('3. Main assembly jar') { + steps { + setScanningStages("main", 3) } } - - } post { always { From f0bcf84e8be15da4334a48511dc5c45c0f3a3606 Mon Sep 17 00:00:00 2001 From: Marek Novotny Date: Tue, 12 Sep 2023 17:44:38 +0200 Subject: [PATCH 4/5] [GH-15744] Address CVE-2017-12197 by Upgrading libpam4j --- h2o-jaas-pam/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h2o-jaas-pam/build.gradle b/h2o-jaas-pam/build.gradle index 8900b5a0336c..1860495d8129 100644 --- a/h2o-jaas-pam/build.gradle +++ b/h2o-jaas-pam/build.gradle @@ -5,5 +5,5 @@ description = "JAAS PAM Module" dependencies { - api "org.kohsuke:libpam4j:1.8" + api "org.kohsuke:libpam4j:1.11" } From 45b6e22fef406a41774d16500a043e6481abe28a Mon Sep 17 00:00:00 2001 From: Marek Novotny Date: Tue, 12 Sep 2023 18:43:45 +0200 Subject: [PATCH 5/5] [GH-15750] Upgrade json-smart to Address CVE-2023-1370 in Main Standalone Jar --- h2o-assemblies/main/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/h2o-assemblies/main/build.gradle b/h2o-assemblies/main/build.gradle index ec63f9c00e73..b0579aa4b1b0 100644 --- a/h2o-assemblies/main/build.gradle +++ b/h2o-assemblies/main/build.gradle @@ -51,6 +51,9 @@ dependencies { api('com.fasterxml.jackson.core:jackson-databind:2.13.4.2') { because 'Fixes CVE-2022-42003' } + api('net.minidev:json-smart:2.4.10') { + because 'Fixes CVE-2023-1370' + } api('com.google.guava:guava:32.0.1-jre') { because 'Fixes CVE-2023-2976' because 'Fixes CVE-2020-8908'