Add ppc64le & multiarch support

.travis.yml

@@ -43,6 +43,7 @@ script:
   - make $MAKE_ARGS gen-certs
   - make $MAKE_ARGS build
   - make $MAKE_ARGS docker-build
+  - make $MAKE_ARGS docker-tag-local
   - make $MAKE_ARGS create-volumes
   # deploy services
   - make $MAKE_ARGS deploy

Makefile

@@ -58,6 +58,10 @@ ifeq ($(IMAGE_NAME),databroker_s3)
     IMAGE_DIR := databroker/s3
 endif
 
+TARGET ?= $(shell uname -m)
+ARCHS ?= x86_64 ppc64le
+
+IMAGE_NAME_TARGET = -$(TARGET)
 
 # Main targets
 
@@ -75,27 +79,59 @@ build: $(addprefix build-, $(SERVICES))
 docker-build-images: $(addprefix docker-build-, $(IMAGES))
 
 docker-build:     ## Build the Docker images for all services
-docker-build: docker-build-images docker-build-ui docker-build-logcollectors
+docker-build: docker-build-base docker-build-images docker-build-ui docker-build-logcollectors
+
+docker-create-manifest:
+	@if [ "${DOCKER_REPO}" = "docker.io" ]; then \
+		docker login --username="${DOCKER_REPO_USER}" --password="${DOCKER_REPO_PASS}"; \
+	else \
+		docker login --username="${DOCKER_REPO_USER}" --password="${DOCKER_REPO_PASS}" https://${DOCKER_REPO}; \
+	fi \
+	for image in $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep -e '${DOCKER_REPO}\/${DOCKER_NAMESPACE}.*-${TARGET}:${IMAGE_TAG}' | grep -v '<none>'); do \
+                manifest_list=`echo $$image | sed -e "s/\(.*\)-${TARGET}\(:.*\)$$/\1\2/g"`; \
+		echo "Creating manifest for $manifest_list"; \
+		AMEND=""; \
+		docker manifest inspect $$manifest_list > /dev/null 2>&1; \
+		if [ $$? -eq 0 ]; then \
+			AMEND="--amend"; \
+		fi; \
+		# lookup for other architectures \
+		other_arch_images=""; \
+		for arch in ${ARCHS}; do \
+			if [ "$$arch" != "${TARGET}" ]; then \
+				arch_image=`echo $$image | sed -e "s/\(.*\)-${TARGET}\(:.*\)$$/\1\-$$arch\2/g"`; \
+				other_arch_images="$$other_arch_image $$arch_image"; \
+			fi; \
+		done; \
+		docker manifest create $$AMEND $$manifest_list $$image $$other_arch_images ${DOCKER_CREATE_MANIFEST_OPTS}; \
+		docker manifest push $$manifest_list ${DOCKER_CREATE_MANIFEST_OPTS}; \
+	done
+
+docker-tag-local:
+	if [ "$(VM_TYPE)" = "minikube" ]; then \
+                        eval $(minikube docker-env); \
+	fi; \
+	for image in $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep -e '${DOCKER_NAMESPACE}.*\-${TARGET}:${IMAGE_TAG}' | grep -v '<none>'); do \
+		image_name=`echo $$image | sed -e "s/\(.*\)-${TARGET}\(:.*\)$$/\1\2/g"`; \
+		echo "docker tag $$image $$image_name"; \
+		docker tag $$image $$image_name; \
+	done \
 
 docker-push:
-	@if [[ -z "${DOCKER_REPO_USER}" ]] || [[ -z "${DOCKER_REPO_PASS}" ]] ; then \
+	@if [ -z "${DOCKER_REPO_USER}" ] || [ -z "${DOCKER_REPO_PASS}" ] ; then \
 		echo "Please define DOCKER_REPO_USER and DOCKER_REPO_PASS."; \
 		exit 1; \
 	else \
 		if [ "${DOCKER_REPO}" = "docker.io" ]; then \
 			docker login --username="${DOCKER_REPO_USER}" --password="${DOCKER_REPO_PASS}"; \
-			for i in $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${DOCKER_NAMESPACE} | grep :${IMAGE_TAG} | grep -v '<none>'); do \
-				echo "docker push $$i"; \
-				docker push $$i; \
-			done; \
 		else \
 			docker login --username="${DOCKER_REPO_USER}" --password="${DOCKER_REPO_PASS}" https://${DOCKER_REPO}; \
-			for i in $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${DOCKER_REPO}/${DOCKER_NAMESPACE} | grep :${IMAGE_TAG} | grep -v '<none>'); do \
-				echo "docker push $$i"; \
-				docker push $$i; \
-			done; \
 		fi; \
-	fi;
+		for i in $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep -e '${DOCKER_REPO}\/${DOCKER_NAMESPACE}.*\-${TARGET}:${IMAGE_TAG}' | grep -v '<none>'); do \
+			echo "docker push $$i"; \
+			docker push $$i; \
+		done; \
+	fi
 
 # TODO: setup-registry
 
@@ -435,14 +471,15 @@ $(addprefix docker-build-, $(IMAGES)): docker-build-%: %
 docker-build-ui:
 	mkdir -p build; test -e dashboard || (git clone $(UI_REPO) build/ffdl-ui; ln -s build/ffdl-ui dashboard)
 	(cd dashboard && (if [ "$(VM_TYPE)" = "minikube" ]; then eval $$(minikube docker-env); fi; \
-		docker build -q -t $(DOCKER_REPO)/$(DOCKER_NAMESPACE)/$(IMAGE_NAME_PREFIX)ui:$(IMAGE_TAG) .; \
-		(test ! `which docker-squash` || docker-squash -t $(DOCKER_REPO)/$(DOCKER_NAMESPACE)/$(IMAGE_NAME_PREFIX)ui $(DOCKER_REPO)/$(DOCKER_NAMESPACE)/$(IMAGE_NAME_PREFIX)ui)))
+		UI_IMAGE_NAME=$(DOCKER_REPO)/$(DOCKER_NAMESPACE)/$(IMAGE_NAME_PREFIX)ui$(IMAGE_NAME_TARGET); \
+		docker build -q --build-arg ARCH=$(TARGET) -t $$UI_IMAGE_NAME:$(IMAGE_TAG) .; \
+		(test ! `which docker-squash` || docker-squash -t $$UI_IMAGE_NAME:$(IMAGE_TAG) $$UI_IMAGE_NAME)))
 
 docker-build-base:
 	if [ "$(VM_TYPE)" = "minikube" ]; then \
 		eval $$(minikube docker-env); \
 	fi; \
-	(cd etc/dlaas-service-base; make build)
+	(cd etc/dlaas-service-base; REPO=${DOCKER_REPO}/ffdl make build)
 
 docker-build-logcollectors:
 	if [ "$(VM_TYPE)" = "minikube" ]; then \
@@ -646,12 +683,12 @@ test-s3:
 	(cd ./$(SERVICE_NAME)/ && (test ! -e main.go || CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -a -installsuffix cgo -o bin/$(BINARY_NAME)))
 
 .docker-build:
-	(full_img_name=$(IMAGE_NAME_PREFIX)$(IMAGE_NAME); \
+	(full_img_name=$(IMAGE_NAME_PREFIX)$(IMAGE_NAME)$(IMAGE_NAME_TARGET); \
 		cd ./$(IMAGE_DIR)/ && ( \
 		if [ "$(VM_TYPE)" = "minikube" ]; then \
 			eval $$(minikube docker-env); \
 		fi; \
-		docker build -q -t $(DOCKER_REPO)/$(DOCKER_NAMESPACE)/$$full_img_name:$(IMAGE_TAG) .))
+		docker build -q --build-arg ARCH=$(TARGET) -t $(DOCKER_REPO)/$(DOCKER_NAMESPACE)/$$full_img_name:$(IMAGE_TAG) .))
 
 kubernetes-ip:
 	@if [ "$$CI" = "true" ]; then kubectl get nodes -o jsonpath='{ .items[0].status.addresses[?(@.type=="InternalIP")].address }'; \

dashboard/Dockerfile

@@ -1,8 +1,20 @@
 FROM node:8-alpine
 
-# install chromium for headless browser tests
-ENV CHROME_BIN=/usr/bin/chromium-browser
-RUN apk add --no-cache chromium udev ttf-freefont
+ARG ARCH=x86_64
+
+# install chromium/firefox for headless browser tests
+ENV CHROME_BIN=/usr/bin/browser
+
+RUN case "${ARCH}" in                                      \
+        amd64)                                             \
+          apk add --no-cache chromium udev ttf-freefont;   \
+          ln -s /usr/bin/chromium-browser /usr/bin/browser \
+          ;;                                               \
+        ppc64le)                                           \
+          apk add --no-cache firefox-esr udev ttf-freefont python build-base; \
+          ln -s /usr/bin/firefox /usr/bin/browser          \
+          ;;                                               \
+    esac
 
 # set workdir
 WORKDIR /ffdl/ffdl-dashboard

etc/dlaas-service-base/Dockerfile.alpine

@@ -15,7 +15,9 @@
 #
 
 
-FROM gliderlabs/alpine:3.3
+FROM alpine:3.7
+
+ARG ARCH=x86_64
 
 RUN apk add --update ca-certificates curl drill
 

etc/dlaas-service-base/Dockerfile.ubuntu

@@ -17,6 +17,8 @@
 
 FROM ubuntu:16.04
 
+ARG ARCH=x86_64
+
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update && apt-get install -qq -y ca-certificates curl ldnsutils
 

etc/dlaas-service-base/Makefile

@@ -15,11 +15,18 @@
 #
 
 
-REPO=docker.io/ffdl
+REPO?=docker.io/ffdl
 IMAGE_NAME=dlaas-service-base
-IMAGE_TAG_ALPINE?=alpine3.3
+IMAGE_TAG_ALPINE?=alpine3.7
 IMAGE_TAG_UBUNTU?=ubuntu16.04
 
+TARGET ?= $(shell uname -m)
+ARCHS ?= x86_64 ppc64le
+
+IMAGE_NAME_TARGET = -$(TARGET)
+
+SERVICE_BASE_IMAGE_NAME = ${IMAGE_NAME}${IMAGE_NAME_TARGET}
+
 all: build push
 
 build-deps:
@@ -30,17 +37,17 @@ build: build-deps build-ubuntu build-alpine
 push: push-alpine push-ubuntu
 
 build-alpine:
-	docker build -q -f Dockerfile.alpine -t ${IMAGE_NAME}:${IMAGE_TAG_ALPINE} .
-	docker tag ${IMAGE_NAME}:${IMAGE_TAG_ALPINE} ${REPO}/${IMAGE_NAME}:${IMAGE_TAG_ALPINE}
+	docker build -q -f Dockerfile.alpine --build-arg ARCH=$(TARGET) -t ${IMAGE_NAME}:${IMAGE_TAG_ALPINE} .
+	docker tag ${IMAGE_NAME}:${IMAGE_TAG_ALPINE} ${REPO}/${SERVICE_BASE_IMAGE_NAME}:${IMAGE_TAG_ALPINE}
 
 build-ubuntu:
-	docker build -q -f Dockerfile.ubuntu -t ${IMAGE_NAME}:${IMAGE_TAG_UBUNTU} .
-	docker tag ${IMAGE_NAME}:${IMAGE_TAG_UBUNTU} ${REPO}/${IMAGE_NAME}:${IMAGE_TAG_UBUNTU}
+	docker build -q -f Dockerfile.ubuntu --build-arg ARCH=$(TARGET) -t ${IMAGE_NAME}:${IMAGE_TAG_UBUNTU} .
+	docker tag ${IMAGE_NAME}:${IMAGE_TAG_UBUNTU} ${REPO}/${SERVICE_BASE_IMAGE_NAME}:${IMAGE_TAG_UBUNTU}
 
 push-alpine:
-	docker push ${REPO}/${IMAGE_NAME}:${IMAGE_TAG_ALPINE}
+	docker push ${REPO}/${SERVICE_BASE_IMAGE_NAME}:${IMAGE_TAG_ALPINE}
 
 push-ubuntu:
-	docker push ${REPO}/${IMAGE_NAME}:${IMAGE_TAG_UBUNTU}
+	docker push ${REPO}/${SERVICE_BASE_IMAGE_NAME}:${IMAGE_TAG_UBUNTU}
 
 .PHONY: build push clean

etc/examples/tf-model/manifest.yml

@@ -15,7 +15,7 @@ data_stores:
     training_results:
       container: tf_trained_model
     connection:
-      auth_url: http://s3.default.svc.cluster.local
+      auth_url: http://10.0.51.5:30851
       user_name: test
       password: test
 

lcm/controller/Dockerfile

@@ -17,14 +17,17 @@
 
 FROM dlaas-service-base:ubuntu16.04
 
+ARG ARCH=x86_64
+
 # Set environment variables.
 ENV ETCD_VERSION=3.3.3
 
 # Install etcdctl from repository.
-RUN apt-get install curl && \
-    curl -LOks https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz && \
-    tar zxvf etcd-v${ETCD_VERSION}-linux-amd64.tar.gz && \
-    cp etcd-v${ETCD_VERSION}-linux-amd64/etcdctl /usr/bin/etcdctl && \
+RUN if [ "${ARCH}" = "x86_64" ]; then TARGET="amd64"; else TARGET=${ARCH}; fi;  \
+    apt-get install curl && \
+    curl -LOks https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-${TARGET}.tar.gz && \
+    tar zxvf etcd-v${ETCD_VERSION}-linux-${TARGET}.tar.gz && \
+    cp etcd-v${ETCD_VERSION}-linux-${TARGET}/etcdctl /usr/bin/etcdctl && \
     rm -rf etcd-v* && \
     chmod +x /usr/bin/etcdctl
 

lcm/controller/Makefile

@@ -22,14 +22,21 @@ IMAGE_TAG ?= user-$(WHOAMI)
 
 STATE_MACHINE_PDF=docs/state-machine.pdf
 
+TARGET ?= $(shell uname -m)
+ARCHS ?= x86_64 ppc64le
+
+IMAGE_NAME_TARGET = -$(TARGET)
+
+IMAGE_NAME ?= ${CONTROLLER_IMAGE_NAME}${IMAGE_NAME_TARGET}:${IMAGE_TAG}
+
 all: build push
 
 build: clean
-	docker build --label git-commit=$(shell git rev-list -1 HEAD) -t ${CONTROLLER_IMAGE_NAME}:${IMAGE_TAG} -f Dockerfile .
-	docker tag ${CONTROLLER_IMAGE_NAME}:${IMAGE_TAG} ${REPO}/${CONTROLLER_IMAGE_NAME}:${IMAGE_TAG}
+	docker build --label git-commit=$(shell git rev-list -1 HEAD) --build-arg ARCH=$(TARGET) -t ${IMAGE_NAME} -f Dockerfile .
+	docker tag ${IMAGE_NAME} ${REPO}/${IMAGE_NAME}
 
 push:
-	docker push ${REPO}/${CONTROLLER_IMAGE_NAME}:${IMAGE_TAG}
+	docker push ${REPO}/${IMAGE_NAME}
 
 docs:
 	cat docs/state-machine.dot |docker run -i --entrypoint /usr/bin/dot andrewshawcare/graphviz -T pdf > ${STATE_MACHINE_PDF}

lcm/service/lcm/constants.go

@@ -81,7 +81,7 @@ const (
 	storeResultsMilliCPU=20
 	storeResultsMemInMB=100
 	loadModelMilliCPU=20
-	loadModelMemInMB=50
+	loadModelMemInMB=100
 	loadTrainingDataMilliCPU=20
 	// FfDL Change: parameterized
 	//loadTrainingDataMemInMB=100

metrics/log_collectors/emetrics_file/Dockerfile

@@ -16,6 +16,8 @@
 
 FROM ubuntu:16.04
 
+ARG ARCH=x86_64
+
 # TODO: Just use community Python 3.6 image per conversation with Atin.
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common
@@ -30,16 +32,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 		mlocate \
 		wget \
 		gcc \
+                g++ \
 	&& rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_VERSION 3.6.3
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     gcc \
     make \
-    musl-dev \
     "python3.6" \
     python3-pip \
+    python3-dev \
+    "python3.6-dev" \
+    python3-setuptools \
 	cron
 
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 1

metrics/log_collectors/emetrics_file/Makefile

@@ -21,7 +21,12 @@ [email protected]:deep-learning-platform/grpc-parameter-server.git
 WHOAMI ?= $(shell whoami)
 IMAGE_TAG ?= user-$(WHOAMI)
 
-IMAGE_NAME?=${DOCKER_REPO}/${DOCKER_NAMESPACE}/emetrics_file_extractor:${IMAGE_TAG}
+TARGET ?= $(shell uname -m)
+ARCHS ?= x86_64 ppc64le
+
+IMAGE_NAME_TARGET := -$(TARGET)
+
+IMAGE_NAME?=${DOCKER_REPO}/${DOCKER_NAMESPACE}/emetrics_file_extractor${IMAGE_NAME_TARGET}:${IMAGE_TAG}
 
 DOCKER_OPTIONS ?=
 
@@ -30,7 +35,7 @@ all: build push
 build:
 	echo "Building ${IMAGE_NAME}"
 	cp -r ../training_data_service_client .
-	docker build -q ${DOCKER_OPTIONS} -t ${IMAGE_NAME} -f Dockerfile .
+	docker build -q ${DOCKER_OPTIONS} -t ${IMAGE_NAME} --build-arg ARCH=$(TARGET) -f Dockerfile .
 	rm -r training_data_service_client
 	echo "Success building ${IMAGE_NAME}"
 

metrics/log_collectors/regex_extractor/Dockerfile

@@ -16,6 +16,8 @@
 
 FROM ubuntu:16.04
 
+ARG ARCH=x86_64
+
 # TODO: Just use community Python 3.6 image per conversation with Atin.
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common
@@ -29,16 +31,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 		mlocate \
 		wget \
 		gcc \
+                g++ \
 	&& rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_VERSION 3.6.3
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     gcc \
     make \
-    musl-dev \
     "python3.6" \
     python3-pip \
+    python3-dev \
+    "python3.6-dev" \
+    python3-setuptools \
 	cron
 
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 1
@@ -74,4 +79,4 @@ RUN mkdir -p /scripts/log_collectors/training_data_service_client/certs
 
 WORKDIR /scripts
 
-RUN echo python=python3 > /root/.bashrc
+RUN echo python=python3 > /root/.bashrc

metrics/log_collectors/regex_extractor/Makefile

@@ -20,15 +20,20 @@ [email protected]:deep-learning-platform/grpc-parameter-server.git
 WHOAMI ?= $(shell whoami)
 IMAGE_TAG ?= user-$(WHOAMI)
 
-IMAGE_NAME?=${DOCKER_REPO}/${DOCKER_NAMESPACE}/regex_extractor:${IMAGE_TAG}
+TARGET ?= $(shell uname -m)
+ARCHS ?= x86_64 ppc64le
+
+IMAGE_NAME_TARGET := -$(TARGET)
+
+IMAGE_NAME?=${DOCKER_REPO}/${DOCKER_NAMESPACE}/regex_extractor${IMAGE_NAME_TARGET}:${IMAGE_TAG}
 
 DOCKER_OPTIONS ?=
 
 all: build push
 
 build:
 	cp -r ../training_data_service_client .
-	docker build -q ${DOCKER_OPTIONS} -t ${IMAGE_NAME} -f Dockerfile .
+	docker build -q ${DOCKER_OPTIONS} -t ${IMAGE_NAME} --build-arg ARCH=$(TARGET) -f Dockerfile .
 	rm -r training_data_service_client
 
 push: