Skip to content

Commit ab3014c

Browse files
mn-mikkemn-mikke
authored
[SW-2504] Python Scoring Package (#2426)
* [SW-2504] Python Scoring Package * Remove extra initializer * copy parts from scoring-py to py * releasing the new package * fix tests * extend mojo predictions test * remove duplicities * update testing pipeline * missing nl * install numpy for tests * Exclude py scoring tests from jar test * Make difference between a reference to a full assembly and a reference to a scoring assembly * Update readme rst. * Fix readme rst.
1 parent 0c02960 commit ab3014c

File tree

80 files changed

+1139
-85
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+1139
-85
lines changed

‎build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ ext {
7474
project(':sparkling-water-doc'),
7575
project(':sparkling-water-booklet')
7676
]
77-
pythonProjects = [project(':sparkling-water-py')]
77+
pythonProjects = [project(':sparkling-water-py'), project(':sparkling-water-py-scoring')]
7878
rProjects = [project(':sparkling-water-r'), project(':sparkling-water-r-cran')]
7979
docProjects = [project(':sparkling-water-doc')]
8080

‎ci/Jenkinsfile-release

Lines changed: 47 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -275,25 +275,30 @@ def buildConda() {
275275
return { params ->
276276
stage('Build Conda Packages') {
277277
if (params.buildConda.toBoolean()) {
278-
dir("py/build/conda") {
279-
def pythonVersions = ['2.7', '3.6']
280-
for (pyVersion in pythonVersions) {
281-
sh """
282-
. /envs/h2o_env_python3.6/bin/activate
283-
284-
conda build h2o_pysparkling_${params.spark} --output-folder "." --no-anaconda-upload --py ${pyVersion}
285-
286-
PACKAGE_PATH=\$(conda build h2o_pysparkling_${params.spark} --py ${pyVersion} --output-folder "." --output | tail -1)
287-
CURRENT_PLATFORM=\$(basename \$(dirname \$PACKAGE_PATH))
288-
mkdir -p ../../../dist/build/dist/py/conda/\$CURRENT_PLATFORM
289-
cp \$PACKAGE_PATH ../../../dist/build/dist/py/conda/\$CURRENT_PLATFORM/
290-
291-
conda convert \$PACKAGE_PATH -p linux-32 -o ../../../dist/build/dist/py/conda/
292-
conda convert \$PACKAGE_PATH -p linux-64 -o ../../../dist/build/dist/py/conda/
293-
conda convert \$PACKAGE_PATH -p win-32 -o ../../../dist/build/dist/py/conda/
294-
conda convert \$PACKAGE_PATH -p win-64 -o ../../../dist/build/dist/py/conda/
295-
conda convert \$PACKAGE_PATH -p osx-64 -o ../../../dist/build/dist/py/conda/
296-
"""
278+
def packages = [
279+
[path: "py/build/conda", name: "h2o_pysparkling_${params.spark}"],
280+
[path: "py-scoring/build/conda", name: "h2o_pysparkling_scoring_${params.spark}"]]
281+
for (packageDetails in packages) {
282+
dir(packageDetails.path) {
283+
def pythonVersions = ['2.7', '3.6']
284+
for (pyVersion in pythonVersions) {
285+
sh """
286+
. /envs/h2o_env_python3.6/bin/activate
287+
288+
conda build ${packageDetails.name} --output-folder "." --no-anaconda-upload --py ${pyVersion}
289+
290+
PACKAGE_PATH=\$(conda build ${packageDetails.name} --py ${pyVersion} --output-folder "." --output | tail -1)
291+
CURRENT_PLATFORM=\$(basename \$(dirname \$PACKAGE_PATH))
292+
mkdir -p ../../../dist/build/dist/py/conda/\$CURRENT_PLATFORM
293+
cp \$PACKAGE_PATH ../../../dist/build/dist/py/conda/\$CURRENT_PLATFORM/
294+
295+
conda convert \$PACKAGE_PATH -p linux-32 -o ../../../dist/build/dist/py/conda/
296+
conda convert \$PACKAGE_PATH -p linux-64 -o ../../../dist/build/dist/py/conda/
297+
conda convert \$PACKAGE_PATH -p win-32 -o ../../../dist/build/dist/py/conda/
298+
conda convert \$PACKAGE_PATH -p win-64 -o ../../../dist/build/dist/py/conda/
299+
conda convert \$PACKAGE_PATH -p osx-64 -o ../../../dist/build/dist/py/conda/
300+
"""
301+
}
297302
}
298303
}
299304
}
@@ -345,13 +350,15 @@ def publishToPipy() {
345350
return { params ->
346351
stage('Publish to PiPy') {
347352
if (params.publishToPiPy.toBoolean()) {
348-
dir("py/build/pkg") {
349-
params.commons.withPipyCredentials {
350-
sh """
351-
. /envs/h2o_env_python3.6/bin/activate
352-
python setup.py sdist
353-
twine upload dist/* -u $PIPY_USERNAME -p $PIPY_PASSWORD
354-
"""
353+
for (projectName in ["py", "py-scoring"]) {
354+
dir("$projectName/build/pkg") {
355+
params.commons.withPipyCredentials {
356+
sh """
357+
. /envs/h2o_env_python3.6/bin/activate
358+
python setup.py sdist
359+
twine upload dist/* -u $PIPY_USERNAME -p $PIPY_PASSWORD
360+
"""
361+
}
355362
}
356363
}
357364
}
@@ -367,10 +374,10 @@ def publishCondaArtifact(arch, pkgName) {
367374
}
368375

369376

370-
def getCondaPkgName(params, pyVersion) {
377+
def getCondaPkgName(pyVersion, packageName) {
371378
return sh(returnStdout: true, script:
372379
"""
373-
CONDA_PKG_CURRENT_ARCH_PATH=\$(conda build h2o_pysparkling_${params.spark} --py ${pyVersion} --output-folder "." --output | tail -1)
380+
CONDA_PKG_CURRENT_ARCH_PATH=\$(conda build ${packageName} --py ${pyVersion} --output-folder "." --output | tail -1)
374381
basename \$CONDA_PKG_CURRENT_ARCH_PATH
375382
""").trim()
376383
}
@@ -379,13 +386,18 @@ def publishToConda() {
379386
return { params ->
380387
stage('Publish to Conda') {
381388
if (params.buildConda.toBoolean() && params.publishConda.toBoolean()) {
382-
dir("py/build/conda") {
383-
params.commons.withCondaCredentials {
384-
def pythonVersions = ['2.7', '3.6']
385-
for (pyVersion in pythonVersions) {
386-
def pkgName = getCondaPkgName(params, pyVersion)
387-
for (arch in ['osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64']) {
388-
publishCondaArtifact(arch, pkgName)
389+
def packages = [
390+
[path: "py/build/conda", name: "h2o_pysparkling_${params.spark}"],
391+
[path: "py-scoring/build/conda", name: "h2o_pysparkling_scoring_${params.spark}"]]
392+
for (packageDetails in packages) {
393+
dir(packageDetails.path) {
394+
params.commons.withCondaCredentials {
395+
def pythonVersions = ['2.7', '3.6']
396+
for (pyVersion in pythonVersions) {
397+
def pkgName = getCondaPkgName(pyVersion, packageDetails.name)
398+
for (arch in ['osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64']) {
399+
publishCondaArtifact(arch, pkgName)
400+
}
389401
}
390402
}
391403
}

‎ci/sparklingWaterPipeline.groovy

Lines changed: 13 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ def unitTests() {
257257
try {
258258
config.commons.withDAICredentials {
259259
sh """
260-
${getGradleCommand(config)} test -x :sparkling-water-r:test -x :sparkling-water-py:test -x integTest -PbackendMode=${config.backendMode}
260+
${getGradleCommand(config)} test -x :sparkling-water-r:test -x :sparkling-water-py:test -x :sparkling-water-py-scoring:test -x integTest -PbackendMode=${config.backendMode}
261261
"""
262262
}
263263
} finally {
@@ -275,30 +275,19 @@ def unitTests() {
275275

276276
def pyUnitTests() {
277277
return { config ->
278-
stage('QA: PyUnit Tests 3.6 - ' + config.backendMode) {
279-
if (config.runPyUnitTests.toBoolean()) {
280-
try {
281-
config.commons.withDAICredentials {
282-
sh """
283-
${getGradleCommand(config)} :sparkling-water-py:test -PpythonPath=/envs/h2o_env_python3.6/bin -PpythonEnvBasePath=/home/jenkins/.gradle/python -x integTest -PbackendMode=${config.backendMode}
284-
"""
285-
}
286-
} finally {
287-
arch '**/build/*tests.log,**/*.log, **/out.*, **/*py.out.txt, **/stdout, **/stderr, **/build/**/*log*, **/build/reports/'
288-
}
289-
}
290-
}
291-
292-
stage('QA: PyUnit Tests 2.7 - ' + config.backendMode) {
293-
if (config.runPyUnitTests.toBoolean()) {
294-
try {
295-
config.commons.withDAICredentials {
296-
sh """
297-
${getGradleCommand(config)} :sparkling-water-py:test -PpythonPath=/envs/h2o_env_python2.7/bin -PpythonEnvBasePath=/home/jenkins/.gradle/python -x integTest -PbackendMode=${config.backendMode}
298-
"""
278+
for (pythonProject in ["py", "py-scoring"]) {
279+
for (pythonVersion in ["2.7", "3.6"]) {
280+
stage("QA: PyUnit Tests ${pythonVersion} - ${pythonProject} - ${config.backendMode}") {
281+
if (config.runPyUnitTests.toBoolean()) {
282+
try {
283+
config.commons.withDAICredentials {
284+
sh "${getGradleCommand(config)} :sparkling-water-${pythonProject}:test " +
285+
"-PpythonPath=/envs/h2o_env_python${pythonVersion}/bin -PpythonEnvBasePath=/home/jenkins/.gradle/python -x integTest -PbackendMode=${config.backendMode}"
286+
}
287+
} finally {
288+
arch '**/build/*tests.log,**/*.log, **/out.*, **/*py.out.txt, **/stdout, **/stderr, **/build/**/*log*, **/build/reports/'
289+
}
299290
}
300-
} finally {
301-
arch '**/build/*tests.log,**/*.log, **/out.*, **/*py.out.txt, **/stdout, **/stderr, **/build/**/*log*, **/build/reports/'
302291
}
303292
}
304293
}

‎dist/build.gradle

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ def distTaskDependencies = [
124124
":sparkling-water-assembly-extensions:shadowJar",
125125
":sparkling-water-templates:build",
126126
":sparkling-water-py:distPython",
127+
":sparkling-water-py-scoring:distPython",
127128
":sparkling-water-r:createCranRepo",
128129
":sparkling-water-core:publish",
129130
":sparkling-water-doc:publish",
@@ -173,7 +174,8 @@ task copyFilesForZipDistribution {
173174

174175
createSupportedHadoopFile("$zipDir")
175176

176-
copyAndKeepPath("py/build/dist/h2o_pysparkling_$sparkMajorVersion-${version}.zip", "$zipDir")
177+
copySingle("py/build/dist/", "h2o_pysparkling_$sparkMajorVersion-${version}.zip", "$zipDir/py")
178+
copySingle("py-scoring/build/dist/", "h2o_pysparkling_scoring_$sparkMajorVersion-${version}.zip", "$zipDir/py")
177179
copyAndKeepPath("LICENSE", "$zipDir")
178180
copyAndKeepPath("README.rst", "$zipDir")
179181
copyAndSubstituteGradleProperties()
@@ -207,7 +209,8 @@ task dist(dependsOn: zipDistribution) {
207209
def buildTimeLocal = new Date().toString()
208210

209211
// Prepare additional files which are accessible directly from the page
210-
copySingle("py/build/dist/", "h2o_pysparkling_$sparkMajorVersion-${version}.zip", "$distDir/py")
212+
copySingle("py/", "h2o_pysparkling_$sparkMajorVersion-${version}.zip", "$distDir/py")
213+
copySingle("py/", "h2o_pysparkling_scoring_$sparkMajorVersion-${version}.zip", "$distDir/py")
211214
copyRecursive("r/build/repo/", "$distDir/R")
212215
copyRecursive("core/build/docs/scaladoc/", "$distDir/scaladoc")
213216
if (makeBooklet.toBoolean()) {

0 commit comments

Comments
 (0)