diff --git a/docker/prisma/Dockerfile.mainjars b/docker/prisma/Dockerfile.mainjars new file mode 100644 index 000000000000..c9eff19411b5 --- /dev/null +++ b/docker/prisma/Dockerfile.mainjars @@ -0,0 +1,2 @@ +FROM alpine:latest +COPY ./h2o-assemblies/main/build/libs/*.jar /tmp/ diff --git a/docker/prisma/Dockerfile.scanningjars b/docker/prisma/Dockerfile.steamjars similarity index 100% rename from docker/prisma/Dockerfile.scanningjars rename to docker/prisma/Dockerfile.steamjars diff --git a/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java b/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java index a476b0327ef4..6103516a88d3 100755 --- a/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java +++ b/h2o-algos/src/test/java/hex/kmeans/KMeansTest.java @@ -37,16 +37,10 @@ private static KMeansModel doSeed( KMeansModel.KMeansParameters parms, long seed return kmm; } - //PUBDEV-871: Double-check the training metrics (gathered by computeStatsFillModel) and the scoring logic by scoring on the training set private static void checkConsistency(KMeansModel kmm) { - //FIXME: TODO: remove this false, and fix the algo! PUBDEV-871 - if (false) { KMeansModel.KMeansParameters parms = kmm._parms; Assert.assertTrue((ArrayUtils.sum(kmm._output._size) - parms.train().numRows()) <= 1); -// Log.info(kmm._output._model_summary); -// Log.info(kmm._output._scoring_history); -// Log.info(((ModelMetricsClustering)kmm._output._training_metrics).createCentroidStatsTable().toString()); kmm.score(parms.train()).delete(); //this scores on the training data and appends a ModelMetrics ModelMetricsClustering mm = (ModelMetricsClustering) ModelMetrics.getFromDKV(kmm, parms.train()); Assert.assertTrue(Arrays.equals(mm._size, ((ModelMetricsClustering) kmm._output._training_metrics)._size)); @@ -56,7 +50,6 @@ private static void checkConsistency(KMeansModel kmm) { Assert.assertTrue(MathUtils.compare(mm._totss, ((ModelMetricsClustering) kmm._output._training_metrics)._totss, 1e-6, 1e-6)); Assert.assertTrue(MathUtils.compare(mm._betweenss, ((ModelMetricsClustering) kmm._output._training_metrics)._betweenss, 1e-6, 1e-6)); Assert.assertTrue(MathUtils.compare(mm._tot_withinss, ((ModelMetricsClustering) kmm._output._training_metrics)._tot_withinss, 1e-6, 1e-6)); - } } @Test public void testIris() { diff --git a/h2o-assemblies/main/build.gradle b/h2o-assemblies/main/build.gradle index 7e0354edeaa6..3feef08d35df 100644 --- a/h2o-assemblies/main/build.gradle +++ b/h2o-assemblies/main/build.gradle @@ -51,6 +51,9 @@ dependencies { api('com.fasterxml.jackson.core:jackson-databind:2.13.4.2') { because 'Fixes CVE-2022-42003' } + api('net.minidev:json-smart:2.4.10') { + because 'Fixes CVE-2023-1370' + } api('org.codehaus.jettison:jettison:1.5.4') { because 'Fixes CVE-2023-1436' because 'Fixes CVE-2022-45693' diff --git a/h2o-jaas-pam/build.gradle b/h2o-jaas-pam/build.gradle index 8900b5a0336c..1860495d8129 100644 --- a/h2o-jaas-pam/build.gradle +++ b/h2o-jaas-pam/build.gradle @@ -5,5 +5,5 @@ description = "JAAS PAM Module" dependencies { - api "org.kohsuke:libpam4j:1.8" + api "org.kohsuke:libpam4j:1.11" } diff --git a/scripts/jenkins/groovy/defineTestStages.groovy b/scripts/jenkins/groovy/defineTestStages.groovy index f09925e5c61b..fb4819f5a1c8 100644 --- a/scripts/jenkins/groovy/defineTestStages.groovy +++ b/scripts/jenkins/groovy/defineTestStages.groovy @@ -440,7 +440,7 @@ def call(final pipelineContext) { ], [ stageName: 'Py3.6 Medium-large', target: 'test-pyunit-medium-large', pythonVersion: '3.6', - timeoutValue: 220, component: pipelineContext.getBuildConfig().COMPONENT_PY + timeoutValue: 300, component: pipelineContext.getBuildConfig().COMPONENT_PY ], [ stageName: 'R3.3 Medium-large', target: 'test-r-medium-large', rVersion: '3.3.3', @@ -546,8 +546,8 @@ def call(final pipelineContext) { ], [ stageName: 'Py3.9 Medium-large', target: 'test-pyunit-medium-large', pythonVersion: '3.9', - timeoutValue: 170, component: pipelineContext.getBuildConfig().COMPONENT_PY - ], + timeoutValue: 300, component: pipelineContext.getBuildConfig().COMPONENT_PY + ], [ // These run with reduced number of file descriptors for early detection of FD leaks stageName: 'XGBoost Stress tests', target: 'test-pyunit-xgboost-stress', pythonVersion: '3.6', timeoutValue: 40, component: pipelineContext.getBuildConfig().COMPONENT_PY, customDockerArgs: [ '--ulimit nofile=150:150' ] diff --git a/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan b/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan index fd5eebaa8161..2a58ae9c62a3 100644 --- a/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan +++ b/scripts/jenkins/jenkinsfiles/Jenkinsfile-PrismaScan @@ -3,15 +3,42 @@ @Library('test-shared-library') _ def dockerImage -def branchOrTag -def steamImage -pipeline { - agent { node { label 'linux&&docker' } } +def setScanningStages(assemblyType, stageIndex) { + def assemblyImage + stage("${stageIndex}.A. Scan ${assemblyType} jar using Prisma") { + script { + branchName = "${env.BRANCH_NAME}".replace('/', '-') + assemblyImage = "h2o-assemblies/${assemblyType}:${BUILD_NUMBER}-${branchName}" + + sh "docker build . -t ${assemblyImage} -f ./docker/prisma/Dockerfile.${assemblyType}jars" - parameters { - string(name: 'BRANCH_OR_TAG', defaultValue: 'master', description: 'Enter branch or tag you want to scan.') + // scan the image + prismaCloudScanImage ca: '', + cert: '', + dockerAddress: 'unix:///var/run/docker.sock', + image: "${assemblyImage}", + key: '', + logLevel: 'info', + podmanPath: '', + project: '', + resultsFile: "prisma-${assemblyType}-scan-results.json", + ignoreImageBuildTime: true + } + } + stage("${stageIndex}.B. Export results for ${assemblyType} jar to CSV") { + withCredentials([usernamePassword(credentialsId: 'twistlock_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "curl -k -u \$USERNAME:\$PASSWORD https://mr-0xz1:8083/api/v1/scans/download?search=${assemblyImage} > ${assemblyImage}.csv" + } + archiveArtifacts artifacts: "${assemblyImage}.csv" } + stage("${stageIndex}.C. Publish report for ${assemblyType} jar") { + prismaCloudPublish resultsFilePattern: "prisma-${assemblyType}-scan-results.json" + } +} + +pipeline { + agent { node { label 'linux&&docker' } } options { ansiColor('xterm') @@ -29,54 +56,28 @@ pipeline { } } - stage('1. Build jar') { + stage('1. Build jars') { steps { script{ dockerImage.inside(){ sh "./gradlew :h2o-assemblies:steam:shadowJar" + sh "./gradlew :h2o-assemblies:main:shadowJar" archiveArtifacts artifacts: "h2o-assemblies/steam/build/libs/*.jar" + archiveArtifacts artifacts: "h2o-assemblies/main/build/libs/*.jar" } } } } - stage('2. Scan jar using Prisma'){ + stage('2. Steam assembly jar') { steps { - script{ - branchOrTag = "${BRANCH_OR_TAG}".replace('/','-') - steamImage = "h2o-assemblies/steam:${BUILD_NUMBER}-${branchOrTag}" - - sh "docker build . -t ${steamImage} -f ./docker/prisma/Dockerfile.scanningjars" - - // scan the image - prismaCloudScanImage ca: '', - cert: '', - dockerAddress: 'unix:///var/run/docker.sock', - image: "${steamImage}", - key: '', - logLevel: 'info', - podmanPath: '', - project: '', - resultsFile: 'prisma-cloud-scan-results.json', - ignoreImageBuildTime:true - } - + setScanningStages("steam", 2) } } - stage('3. Export results to CSV'){ - steps{ - withCredentials([usernamePassword(credentialsId: 'twistlock_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "curl -k -u \$USERNAME:\$PASSWORD https://mr-0xz1:8083/api/v1/scans/download?search=${steamImage} > ${steamImage}.csv" - } - archiveArtifacts artifacts: "${steamImage}.csv" - } - } - stage('4. Publish report'){ - steps{ - prismaCloudPublish resultsFilePattern: 'prisma-cloud-scan-results.json' + stage('3. Main assembly jar') { + steps { + setScanningStages("main", 3) } } - - } post { always {