Merge pull request #282 from Donders-Institute/stager-buildin

Stager buildin

Author: hurngchunlee

Jenkinsfile

@@ -1,3 +1,6 @@
+// STACK_NAME defines the name of the Docker stack deployed to acceptance
+def STACK_NAME="lab-data-streamer"
+
 pipeline {
     agent any
 
@@ -71,26 +74,10 @@ pipeline {
             }
             steps {
 
-                sh 'docker stack rm streamer4user'
+                sh "docker stack rm ${STACK_NAME}"
 
                 sleep(30)
 
-                // Streamer secrets
-                configFileProvider([configFile(fileId: 'streamer_service_config.json', variable: 'STREAMER_SERVICE_CONFIG')]) {
-                    sh 'docker secret rm streamer-service-config.json || true'
-                    sh 'docker secret create streamer-service-config.json $STREAMER_SERVICE_CONFIG'
-                }
-                configFileProvider([configFile(fileId: 'streamer_mailer_config.json', variable: 'STREAMER_MAILER_CONFIG')]) {
-                    sh 'docker secret rm streamer-mailer-config.json || true'
-                    sh 'docker secret create streamer-mailer-config.json $STREAMER_MAILER_CONFIG'
-                }
-
-                // Streamer UI secrets
-                configFileProvider([configFile(fileId: 'streamer_ui_config.json', variable: 'STREAMER_UI_CONFIG')]) {
-                    sh 'docker secret rm streamer-ui-config.json || true'
-                    sh 'docker secret create streamer-ui-config.json $STREAMER_UI_CONFIG'
-                }
-
                 withCredentials([
                     usernamePassword (
                         credentialsId: params.STREAMER_UI_DB_CREDENTIALS,
@@ -116,7 +103,7 @@ pipeline {
 
                     // Use the same approach as for production
                     script {
-                        def statusCode = sh(script: "bash ./docker-deploy-acceptance.sh", returnStatus: true)
+                        def statusCode = sh(script: "bash ./docker-deploy-acceptance.sh ${STACK_NAME}", returnStatus: true)
                         echo "statusCode: ${statusCode}"
                     }
                 }
@@ -133,26 +120,27 @@ pipeline {
                 label 'swarm-manager'
             }
             steps {
-                withDockerContainer(image: 'jwilder/dockerize', args: '--network streamer4user-net') {
+                withDockerContainer(image: 'jwilder/dockerize', args: "--network ${STACK_NAME}_default") {
                     sh (
                         label: 'Waiting for services to become available',
                         script: 'dockerize \
                             -timeout 120s \
                             -wait tcp://service:3001 \
                             -wait tcp://ui-db:5432 \
-                            -wait http://ui:9000'
+                            -wait http://ui:9000 \
+                            -wait http://stager-api-server:8080/v1/ping'
                     )
                 }
             }
             post {
                 failure {
                     sh (
                         label: 'Displaying service status',
-                        script: 'docker stack ps streamer4user'
+                        script: "docker stack ps ${STACK_NAME}"
                     )
                     sh (
                         label: 'Displaying service logs',
-                        script: 'docker stack services --format \'{{.Name}}\' streamer4user | xargs -n 1 docker service logs'
+                        script: "docker stack services --format '{{.Name}}' ${STACK_NAME} | xargs -n 1 docker service logs"
                     )
                 }
             }
@@ -230,7 +218,7 @@ pipeline {
 
     post {
         success {
-            archiveArtifacts "docker-compose.yml, docker-compose.swarm.yml, env.sh"
+            archiveArtifacts "docker-compose.yml, docker-compose.stager.yml, docker-compose.swarm.yml, env.sh"
         }
         always {
             echo 'cleaning'

docker-compose.override.yml

@@ -7,7 +7,6 @@ services:
     ports:
       - ${STREAMER_UI_EXTERNAL_PORT:-9000}:9000
     volumes:
-      - ${STREAMER_SECRETS_DIR}:/opt/streamer-ui-server/config
       - ${STREAMER_UI_LOG_VOL}:/opt/streamer-ui-server/log
   ui-db:
     ports:

docker-compose.stager.yml

@@ -0,0 +1,24 @@
+version: '3.7'
+services:
+
+  # stager task database
+  stager-db:
+    image: redis:alpine
+    user: root
+    command: --appendonly yes
+
+  # stager api server
+  stager-api-server:
+    image: ${DOCKER_REGISTRY:-dccn}/data-stager-api:${STAGER_DOCKER_IMAGE_TAG:-latest}
+    user: root
+    depends_on:
+      - stager-db
+    command: -p 8080 -r redis://stager-db:6379 -c /etc/stager/api-server.yml
+
+  # stager worker
+  stager-worker:
+    image: ${DOCKER_REGISTRY:-dccn}/data-stager-worker:${STAGER_DOCKER_IMAGE_TAG:-latest}
+    user: root
+    depends_on:
+      - stager-api-server
+    command: -r redis://stager-db:6379 -c /etc/stager/worker.yml

docker-compose.swarm.yml

@@ -1,66 +1,74 @@
 version: "3.7"
 services:
+  ## streamer
   service:
     volumes:
       # bind-mount for sss authentication
       - /var/lib/sss/pipes:/var/lib/sss/pipes
       - /var/lib/sss/mc:/var/lib/sss/mc:ro
-    environment:
-      # provides configuration files from docker secrets
-      - STREAMER_SERVICE_CONFIG=/run/secrets/streamer-service-config.json
-      - STREAMER_MAILER_CONFIG=/run/secrets/streamer-mailer-config.json
+      - ${STREAMER_SERVICE_CONFIG}:/opt/streamer/config/default.json
+      - ${STREAMER_MAILER_CONFIG}:/opt/streamer/config/mailer.json
     networks:
       default:
       # join the proxynet with hostname alias
       proxynet:
         aliases:
           - streamer
-    secrets:
-      - streamer-service-config.json
-      - streamer-mailer-config.json
   ui:
-    environment:
-      # provides configuration files from docker secrets
-      - STREAMER_UI_CONFIG=/run/secrets/streamer-ui-config.json
-      - STREAMER_SERVICE_CONFIG=/run/secrets/streamer-service-config.json
     networks:
       default:
       # join the proxynet with hostname alias
       proxynet:
         aliases:
           - streamer-ui
-    secrets:
-      - streamer-service-config.json
-      - streamer-ui-config.json
-      - streamer-ui-adconfig.json
-      - streamer-ui-ldapscert.crt
     ports:
       - ${STREAMER_UI_EXTERNAL_PORT:-9000}:9000
     volumes:
       - ${PROJECT_CEPHFS_VOL}:/project_cephfs
       - ${PROJECT_VOL}:/project
       - ${STREAMER_UI_LOG_VOL}:/opt/streamer-ui-server/log
+      - ${STREAMER_SERVICE_CONFIG}:/opt/streamer-ui-server/config/streamer-service-config.json
+      - ${STREAMER_UI_CONFIG}:/opt/streamer-ui-server/config/streamer-ui-config.json
   ui-db:
-    ports:
-      - ${STREAMER_UI_DB_EXTERNAL_PORT:-9001}:${STREAMER_UI_DB_PORT:-5432}
     volumes:
       - ${STREAMER_UI_DB_DATA_VOL}:/var/lib/postgresql/data
 
+  ## stager
+  stager-db:
+    volumes:
+      - ${STAGER_DB_DATA_VOL}:/data
+
+  # stager api server
+  stager-api-server:
+    volumes:
+      - ${STAGER_API_CONFIG}:/etc/stager/api-server.yml:ro
+      - ${PROJECT_VOL:-/project}:/project
+      - ${PROJECT_CEPHFS_VOL:-/project_cephfs}:/project_cephfs
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "5"
+
+  # stager worker
+  stager-worker:
+    volumes:
+      - ${STAGER_IRODS_ICAT_CERT}:/opt/irods/ssl/icat.pem:ro
+      - ${STAGER_WORKER_CONFIG}:/etc/stager/worker.yml:ro
+      - ${PROJECT_VOL:-/project}:/project
+      - ${PROJECT_CEPHFS_VOL:-/project_cephfs}:/project_cephfs
+    deploy:
+      placement:
+        constraints:
+          - node.labels.IpRange==44
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "5"
+
 networks:
   default:
-    name: streamer4user-net
     attachable: true
   proxynet:
-    external: true
-
-secrets:
-  streamer-service-config.json:
-    external: true
-  streamer-mailer-config.json:
-    external: true
-  streamer-ui-config.json:
-    external: true
-  streamer-ui-adconfig.json:
-    external: true
-  streamer-ui-ldapscert.crt:
-    external: true
+    external: true

docker-deploy-acceptance.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
+[ $# -lt 1 ] && echo "missing Docker stack name" >&2 && exit 1
+
 # export variables defined in env.sh
 set -a && source env.sh && set +a
 
-docker stack up -c docker-compose.yml -c docker-compose.swarm.yml --prune --with-registry-auth --resolve-image always streamer4user
+docker stack up -c docker-compose.yml -c docker-compose.stager.yml -c docker-compose.swarm.yml --prune --with-registry-auth --resolve-image always "$1"

docker-deploy.sh

@@ -3,15 +3,4 @@
 # export variables defined in env.sh
 set -a && source env.sh && set +a
 
-# add secrets, file and secret name will be identical
-for f in `ls $STREAMER_SECRETS_DIR`; do
-    docker secret ls -f name=$f --format "{{.Name}}" | grep $f >/dev/null 2>&1
-    if [ $? -eq 0 ]; then
-        # secret with the same name exists, remove it
-        docker secret rm $f
-    fi
-    # create new secret
-    docker secret create $f secrets/$f
-done
-
-docker stack up -c docker-compose.yml -c docker-compose.swarm.yml --with-registry-auth streamer4user
+docker stack up -c docker-compose.yml -c docker-compose.stager.yml -c docker-compose.swarm.yml --with-registry-auth lab-data-streamer

env.sh

@@ -14,7 +14,7 @@ PROJECT_VOL=./testdata/project
 PROJECT_CEPHFS_VOL=./testdata/project_cephfs
 
 # volume for streamer job
-STREAMER_DB_DATA_VOL=./testdata/redis
+STREAMER_DB_DATA_VOL=./testdata/streamer/db
 
 # volume for streamer log
 STREAMER_SERVICE_LOG_VOL=./testdata/streamer/log
@@ -35,7 +35,6 @@ STREAMER_UI_CRON_VOL=./testdata/ui/cron
 STREAMER_UI_LOG_VOL=./testdata/ui/log
 
 # Configuration files
-STREAMER_SECRETS_DIR=./streamer/config
 STREAMER_SERVICE_CONFIG=./streamer/config/streamer-service-config.json
 STREAMER_MAILER_CONFIG=./streamer/config/streamer-mailer-config.json
 STREAMER_UI_CONFIG=./streamer/config/streamer-ui-config.json
@@ -68,3 +67,10 @@ POSTGRES_PASSWORD=$STREAMER_UI_DB_PASSWORD
 POSTGRES_DATABASE=$STREAMER_UI_DB_NAME
 GRAFANA_USER=grafanareader
 GRAFANA_PASSWORD=grafanareaderpassword
+
+# configuration for stager
+STAGER_DOCKER_IMAGE_TAG=latest
+STAGER_DB_DATA_VOL=./testdata/stager/db
+STAGER_API_CONFIG=./streamer/config/stager-api-server.yml
+STAGER_WORKER_CONFIG=./streamer/config/stager-worker.yml
+STAGER_IRODS_ICAT_CERT=./streamer/config/stager-icat.pem

print_env.sh

@@ -37,7 +37,6 @@ echo "# volume for streamer ui log"
 echo "STREAMER_UI_LOG_VOL=$STREAMER_UI_LOG_VOL"
 echo
 echo "# Configuration files"
-echo "STREAMER_SECRETS_DIR=$STREAMER_SECRETS_DIR"
 echo "STREAMER_SERVICE_CONFIG=$STREAMER_SERVICE_CONFIG"
 echo "STREAMER_MAILER_CONFIG=$STREAMER_MAILER_CONFIG"
 echo "STREAMER_UI_CONFIG=$STREAMER_UI_CONFIG"
@@ -71,3 +70,10 @@ echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD"
 echo "POSTGRES_DATABASE=$POSTGRES_DATABASE"
 echo "GRAFANA_USER=$GRAFANA_USER"
 echo "GRAFANA_PASSWORD=$GRAFANA_PASSWORD"
+echo
+echo "# configuration for stager"
+echo "STAGER_DOCKER_IMAGE_TAG=$STAGER_DOCKER_IMAGE_TAG"
+echo "STAGER_DB_DATA_VOL=$STAGER_DB_DATA_VOL"
+echo "STAGER_API_CONFIG=$STAGER_API_CONFIG"
+echo "STAGER_WORKER_CONFIG=$STAGER_WORKER_CONFIG"
+echo "STAGER_IRODS_ICAT_CERT=$STAGER_IRODS_ICAT_CERT"

streamer-ui/packages/server/routes/stager.js

@@ -1,7 +1,6 @@
 const createError = require("http-errors");
 const path = require("path");
-const { basicAuthString, fetchOnce } = require("./utils");
-const fetch = require("node-fetch");
+const { fetchOnce } = require("./utils");
 
 // location of the streamer-service-config.json file.
 const fconfig = path.join(__dirname, '../config/streamer-service-config.json');
@@ -20,15 +19,13 @@ var _getDac = async function(req, res, next) {
     delete require.cache[require.resolve(fconfig)];
 
     const headers = {
-        'Content-Type': 'application/json',
-        'Authorization': basicAuthString(config.DataStager.username, config.DataStager.password),
+        'Content-Type': 'application/json'
     };
 
     fetchOnce(
-        config.DataStager.url + "/rdm/DAC/project/" + projectId,
+        config.DataStager.url + "/dac/project/" + projectId,
         {
             method: 'GET',
-            credentials: 'include',
             headers,
         },
         1000 * 30,  // timeout after 30 seconds

streamer-ui/packages/server/start-server.sh

@@ -3,16 +3,6 @@
 cd /opt/streamer-ui-server
 
 env
-ls -l /run/secrets
-
-# Check and copy configuration files from secrets
-if [ -f $STREAMER_SERVICE_CONFIG ]; then
-    cp $STREAMER_SERVICE_CONFIG config/streamer-service-config.json
-fi
-
-if [ -f $STREAMER_UI_CONFIG ]; then
-    cp $STREAMER_UI_CONFIG config/streamer-ui-config.json
-fi
 
 ls -l config