[BUILD] Dockerfile and Jenkinsfile revisited (dmlc#2514)

Includes:
  - Dockerfile changes
    - Dockerfile clean up
    - Fix execution privileges of files used from Dockerfile.
    - New Dockerfile entrypoint to replace with_user script
    - Defined a placeholders for CPU testing (script and Dockerfile)
  - Jenkinsfile
    - Jenkins file milestone defined
    - Single source code checkout and propagation via stash/unstash
    - Bash needs to be explicitly used in launching make build, since we need
access to environment
    - Jenkinsfile build factory for cmake and make style of jobs
    - Archivation of artifacts (*.so, *.whl, *.egg) produced by cmake build

Missing:
  - CPU testing
  - Python3 env build and testing

Jenkinsfile

@@ -1,15 +1,54 @@
+#!/usr/bin/groovy
 // -*- mode: groovy -*-
 // Jenkins pipeline
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
-// command to start a docker container
-docker_run = 'tests/ci_build/ci_build.sh'
+// Command to run command inside a docker container
+dockerRun = 'tests/ci_build/ci_build.sh'
 
-// timeout in minutes
-max_time = 60
+def buildMatrix = [
+    [ "enabled": true,  "os" : "linux", "withGpu": true,  "withOmp": true, "pythonVersion": "2.7" ],
+    [ "enabled": true,  "os" : "linux", "withGpu": false, "withOmp": true, "pythonVersion": "2.7" ],
+    [ "enabled": false, "os" : "osx",   "withGpu": false, "withOmp": false, "pythonVersion": "2.7" ],
+]
+
+pipeline {
+    // Each stage specify its own agent
+    agent none
+
+    // Setup common job properties
+    options {
+        ansiColor('xterm')
+        timestamps()
+        timeout(time: 120, unit: 'MINUTES')
+        buildDiscarder(logRotator(numToKeepStr: '10'))
+    }
+
+    // Build stages
+    stages {
+        stage('Get sources') {
+            agent any
+            steps {
+                checkoutSrcs()
+                stash name: 'srcs', excludes: '.git/'
+                milestone label: 'Sources ready', ordinal: 1
+            }
+        }
+        stage('Build & Test') {
+            steps {
+                script {
+                    parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
+                        def buildName = getBuildName(c)
+                        buildFactory(buildName, c)
+                    })
+                }
+            }
+        }
+    }
+}
 
 // initialize source codes
-def init_git() {
+def checkoutSrcs() {
   retry(5) {
     try {
       timeout(time: 2, unit: 'MINUTES') {
@@ -23,33 +62,90 @@ def init_git() {
   }
 }
 
-stage('Build') {
-    node('GPU' && 'linux') {
-      ws('workspace/xgboost/build-gpu-cmake') {
-        init_git()
-  	timeout(time: max_time, unit: 'MINUTES') {
-      		sh "${docker_run} gpu tests/ci_build/build_gpu_cmake.sh"
-	}
-      }
+/**
+ * Creates cmake and make builds
+ */
+def buildFactory(buildName, conf) {
+    def os = conf["os"]
+    def nodeReq = conf["withGpu"] ? "${os} && gpu" : "${os}"
+    def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
+    [ ("cmake_${buildName}") : { buildPlatformCmake("cmake_${buildName}", conf, nodeReq, dockerTarget) },
+      ("make_${buildName}") : { buildPlatformMake("make_${buildName}", conf, nodeReq, dockerTarget) }
+    ]
+}
+
+/**
+ * Build platform and test it via cmake.
+ */
+def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
+    def opts = cmakeOptions(conf)
+    // Destination dir for artifacts
+    def distDir = "dist/${buildName}"
+    // Build node - this is returned result
+    node(nodeReq) {
+        unstash name: 'srcs'
+        echo """
+        |===== XGBoost CMake build =====
+        |  dockerTarget: ${dockerTarget}
+        |  cmakeOpts   : ${opts}
+        |=========================
+        """.stripMargin('|')
+        // Invoke command inside docker
+        sh """
+        ${dockerRun} ${dockerTarget} tests/ci_build/build_via_cmake.sh ${opts}
+        ${dockerRun} ${dockerTarget} tests/ci_build/test_${dockerTarget}.sh
+        ${dockerRun} ${dockerTarget} bash -c "cd python-package; python setup.py bdist_wheel"
+        rm -rf "${distDir}"; mkdir -p "${distDir}/py"
+        cp xgboost "${distDir}"
+        cp -r lib "${distDir}"
+        cp -r python-package/dist "${distDir}/py"
+        """
+        archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
     }
-    node('GPU' && 'linux') {
-      ws('workspace/xgboost/build-gpu-make') {
-        init_git()
-  	timeout(time: max_time, unit: 'MINUTES') {
-      		sh "${docker_run} gpu make PLUGIN_UPDATER_GPU=ON"
-	}
-      }
+}
+
+/**
+ * Build platform via make
+ */
+def buildPlatformMake(buildName, conf, nodeReq, dockerTarget) {
+    def opts = makeOptions(conf)
+    // Destination dir for artifacts
+    def distDir = "dist/${buildName}"
+    // Build node
+    node(nodeReq) {
+        unstash name: 'srcs'
+        echo """
+        |===== XGBoost Make build =====
+        |  dockerTarget: ${dockerTarget}
+        |  makeOpts    : ${opts}
+        |=========================
+        """.stripMargin('|')
+        // Invoke command inside docker
+        sh """
+        ${dockerRun} ${dockerTarget} tests/ci_build/build_via_make.sh ${opts}
+        """
     }
 }
 
+def makeOptions(conf) {
+    return ([
+        conf["withGpu"] ? 'PLUGIN_UPDATER_GPU=ON' : 'PLUGIN_UPDATER_GPU=OFF',
+        conf["withOmp"] ? 'USE_OPENMP=1' : 'USE_OPENMP=0']
+        ).join(" ")
+}
 
-stage('Unit Test') {
-    node('GPU' && 'linux') {
-      ws('workspace/xgboost/unit-test') {
-        init_git()
-  	timeout(time: max_time, unit: 'MINUTES') {
-      		sh "${docker_run} gpu tests/ci_build/test_gpu.ssh"
-	}
-      }
-    }
+
+def cmakeOptions(conf) {
+    return ([
+        conf["withGpu"] ? '-DPLUGIN_UPDATER_GPU:BOOL=ON' : '',
+        conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : '']
+        ).join(" ")
+}
+
+def getBuildName(conf) {
+    def gpuLabel = conf['withGpu'] ? "_gpu" : "_cpu"
+    def ompLabel = conf['withOmp'] ? "_omp" : ""
+    def pyLabel = "_py${conf['pythonVersion']}"
+    return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"
 }
+

tests/ci_build/Dockerfile.cpu

@@ -0,0 +1,49 @@
+FROM ubuntu:14.04
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update -q -y && \
+    apt-get -y dist-upgrade && \
+    apt-get -y --no-install-recommends  install \
+        build-essential \
+        wget \
+        unzip \
+        gfortran \
+        # BLAS
+        libatlas-base-dev \
+        # Python 2
+        python-setuptools \
+        python-pip \
+        python-dev \
+    && \
+    # CMake
+    wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
+    tar -xvzf cmake-3.5.2.tar.gz && \
+    cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
+    # Clean up
+    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
+    apt-get clean && \
+    rm -rf /var/cache/apt/*
+
+
+# Install Python packages
+RUN pip install numpy nose scipy scikit-learn wheel
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/Dockerfile.gpu

@@ -1,16 +1,49 @@
 FROM nvidia/cuda:8.0-devel-ubuntu14.04
 
-RUN apt-get update && apt-get -y upgrade
-# CMAKE
-RUN sudo apt-get install -y build-essential
-RUN apt-get install -y wget
-RUN wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz 
-RUN tar -xvzf cmake-3.5.2.tar.gz 
-RUN cd cmake-3.5.2/ && ./configure && make && sudo make install
-
-# BLAS
-RUN apt-get install -y libatlas-base-dev
-
-# PYTHON2
-RUN apt-get install -y python-setuptools python-pip python-dev unzip gfortran
-RUN pip install numpy nose scipy scikit-learn
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update -q -y && \
+    apt-get -y dist-upgrade && \
+    apt-get -y --no-install-recommends  install \
+        build-essential \
+        wget \
+        unzip \
+        gfortran \
+        # BLAS
+        libatlas-base-dev \
+        # Python 2
+        python-setuptools \
+        python-pip \
+        python-dev \
+    && \
+    # CMake
+    wget http://www.cmake.org/files/v3.5/cmake-3.5.2.tar.gz && \
+    tar -xvzf cmake-3.5.2.tar.gz && \
+    cd cmake-3.5.2/ && ./configure && make && make install && cd ../ && \
+    # Clean up
+    rm -rf cmake-3.5.2/ && rm -rf cmake-3.5.2.tar.gz && \
+    apt-get clean && \
+    rm -rf /var/cache/apt/*
+
+
+# Install Python packages
+RUN pip install numpy nose scipy scikit-learn wheel
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/build_gpu_cmake.sh → tests/ci_build/build_via_cmake.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
+make clean
 mkdir build
 cd build
-cmake .. -DPLUGIN_UPDATER_GPU=ON
+cmake .. "$@"
 make

tests/ci_build/build_via_make.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+make clean
+make "$@"

tests/ci_build/ci_build.sh

@@ -83,29 +83,33 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
 # Convert to all lower-case, as per requirement of Docker image names
 DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
 
-# skip with_the_same_user for non-linux
-uname=`uname`
-if [[ "$uname" == "Linux" ]]; then
-    PRE_COMMAND="tests/ci_build/with_the_same_user"
-else
-    PRE_COMMAND=""
+# Bash on Ubuntu on Windows
+UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
+# MSYS, Git Bash, etc.
+MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
+
+if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then
+    USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}"
 fi
 
 # Print arguments.
-echo "WORKSPACE: ${WORKSPACE}"
-echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
-echo "COMMAND: ${COMMAND[@]}"
-echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
-echo "BUILD_TAG: ${BUILD_TAG}"
-echo "NODE_NAME: ${NODE_NAME}"
-echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}"
-echo "PRE_COMMAND: ${PRE_COMMAND}"
-echo ""
+cat <<EOF
+   WORKSPACE: ${WORKSPACE}
+   CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[*]}
+   COMMAND: ${COMMAND[*]}
+   CONTAINER_TYPE: ${CONTAINER_TYPE}
+   BUILD_TAG: ${BUILD_TAG}
+   NODE_NAME: ${NODE_NAME}
+   DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}
+   USER_IDS: ${USER_IDS}
+EOF
 
 
 # Build the docker container.
 echo "Building container (${DOCKER_IMG_NAME})..."
-docker build -t ${DOCKER_IMG_NAME} \
+# --pull should be default
+docker build \
+    -t "${DOCKER_IMG_NAME}" \
     -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}"
 
 # Check docker build status
@@ -116,20 +120,16 @@ fi
 
 
 # Run the command inside the container.
-echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
+echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..."
 
 # By default we cleanup - remove the container once it finish running (--rm)
 # and share the PID namespace (--pid=host) so the process inside does not have
 # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
 ${DOCKER_BINARY} run --rm --pid=host \
-    -v ${WORKSPACE}:/workspace \
+    -v "${WORKSPACE}":/workspace \
     -w /workspace \
-    -e "CI_BUILD_HOME=${WORKSPACE}" \
-    -e "CI_BUILD_USER=$(id -u -n)" \
-    -e "CI_BUILD_UID=$(id -u)" \
-    -e "CI_BUILD_GROUP=$(id -g -n)" \
-    -e "CI_BUILD_GID=$(id -g)" \
-    ${CI_DOCKER_EXTRA_PARAMS[@]} \
-    ${DOCKER_IMG_NAME} \
-    ${PRE_COMMAND} \
-    ${COMMAND[@]}
+    ${USER_IDS} \
+    "${CI_DOCKER_EXTRA_PARAMS[@]}" \
+    "${DOCKER_IMG_NAME}" \
+    "${COMMAND[@]}"
+