docker process refacto: packaged in Makefile

.dockerignore

@@ -1,3 +1,6 @@
 esdata/
 log/
+pgdata/
+kibana/
+projects/
 

.gitignore

@@ -1,2 +1,8 @@
 esdata
 pgdata
+data
+.lastcommit
+kibana
+log
+projects
+

Makefile

@@ -0,0 +1,120 @@
+export BACKEND := $(shell pwd)
+
+export FRONTEND=${BACKEND}/../frontend
+export UPLOAD=${BACKEND}/data
+export PROJECTS=${BACKEND}/projects
+export MODELS=${BACKEND}/models
+export LOG=${BACKEND}/log
+export DC_DIR=${BACKEND}/docker-components
+export DC_FILE=${DC_DIR}/docker-compose
+
+commit              := $(shell git rev-parse HEAD | cut -c1-8)
+lastcommit          := $(shell touch .lastcommit; cat .lastcommit)
+commit-frontend     := $(shell cd ${FRONTEND}; git rev-parse HEAD | cut -c1-8)
+lastcommit-frontend := $(shell touch ${FRONTEND}/.lastcommit; cat ${FRONTEND}/.lastcommit)
+date                := $(shell date -I)
+
+ES_NODES := 1
+PG := 'postgres'
+DC := 'docker-compose'
+NH := 'nohup'
+
+
+clean:
+	sudo rm -rf ${FRONTEND}/dist
+	sudo mkdir -p ${UPLOAD} ${PROJECTS} ${MODELS}
+
+network-stop:
+	docker network rm matchid
+
+network:
+	@docker network create matchid 2> /dev/null; true
+
+elasticsearch-stop:
+	@echo docker-compose down matchID elasticsearch
+	@${DC} -f ${DC_FILE}-elasticsearch-phonetic.yml down 
+	@${DC} -f ${DC_FILE}-elasticsearch-huge.yml down
+
+elasticsearch: network
+ifeq "$(ES_NODES)" "1"
+	@echo docker-compose up matchID elasticsearch
+	@sudo mkdir -p esdata/node
+	@(${NH} ${DC} -f ${DC_FILE}-elasticsearch-phonetic.yml up > ${LOG}/docker-elasticsearch.log) 2> /dev/null & 
+else
+	@echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes
+	@cat ${DC_FILE}-elasticsearch.yml > ${DC_FILE}-elasticsearch-huge.yml
+	@i=2; $while [$i -le $ES_NODES]; do cat ${DC_FILE}-elasticsearch-node.yml | sed "s/%N/$i" >> ${DC_FILE}-elasticsearch-huge.yml;done
+	@(${NH} ${DC} -f ${DC_FILE}-elasticsearch-huge.yml up > ${LOG}/docker-elasticsearch.log) 2> /dev/null & 
+endif
+	@sleep 2 && tail ${LOG}/docker-elasticsearch.log &
+
+kibana-stop:
+	${DC} -f ${DC_FILE}-kibana.yml down
+kibana: network
+	@echo docker-compose up matchID kibana
+	@(${NH} ${DC} -f ${DC_FILE}-kibana.yml up > ${LOG}/docker-kibana.log) 2> /dev/null & 
+	@sleep 2 && tail ${LOG}/docker-kibana.log &
+
+postgres-stop:
+	${DC} -f ${DC_FILE}-${PG}.yml down
+postgres: network
+	@echo docker-compose up matchID postgres
+	@(${NH} ${DC} -f ${DC_FILE}-${PG}.yml up > ${LOG}/docker-postgres.log) 2> /dev/null & 
+	@sleep 2 && tail ${LOG}/docker-postgres.log &
+
+backend-stop:
+	${DC} down 
+backend: network
+	@echo docker-compose up matchID backend
+	@(${NH} ${DC} up > ${LOG}/docker-backend.log) 2> /dev/null & 
+	@sleep 2 && tail ${LOG}/docker-backend.log &
+
+
+frontend-download:
+	@mkdir -p ${FRONTEND}
+	@cd ${FRONTEND}
+	@git clone https://github.com/matchID-project/frontend . 2> /dev/null; true 
+	@cd ${BACKEND}
+
+frontend-dev: frontend-download network backend elasticsearch kibana postgres
+ifneq "$(commit-frontend)" "$(lastcommit-frontend)"
+	@echo docker-compose up matchID frontend for dev after new commit
+	@${NH} ${DC} -f docker/docker-compose-dev.yml down
+	@(${NH} ${DC} -f docker/docker-compose-dev.yml up --build > ${LOG}/docker-frontend.log) 2> /dev/null & 
+	@echo "${commit-frontend}" > ${FRONTEND}/.lastcommit
+else
+	@echo docker-compose up matchID frontend for dev
+	@(${NH} ${DC} -f docker-compose-dev.yml up > ${LOG}/docker-frontend.log) 2> /dev/null & 
+endif
+	@sleep 2 && tail ${LOG}/docker-frontend.log &
+
+frontend-build: frontend-download network
+ifneq "$(commit-frontend)" "$(lastcommit-frontend)"
+	@echo building matchID frontend after new commit
+	@make clean
+	@echo building frontend in ${FRONTEND}
+	@sudo mkdir -p ${FRONTEND}/dist
+	${DC} -f ${DC_FILE}-build-frontend.yml up --build
+	@echo "${commit-frontend}" > ${FRONTEND}/.lastcommit
+endif	
+
+frontend-stop:
+	${DC} -f ${DC_FILE}-run-frontend.yml down
+
+frontend: frontend-build
+	@echo docker-compose up matchID frontend
+	@(${NH} ${DC} -f ${DC_FILE}-run-frontend.yml up > ${LOG}/docker-frontend.log) 2> /dev/null & 
+	@sleep 2 && tail ${LOG}/docker-frontend.log &
+
+stop: backend-stop elasticsearch-stop kibana-stop postgres-stop 
+	@echo all components stopped
+
+start: frontend backend elasticsearch kibana postgres
+	@sleep 2 && echo all components started, please enter following command to supervise: 
+	@echo tail log/docker-*.log
+
+tuto: 
+	export UPLOAD=../tutorial/data/;
+	export PROJECTS=../tutorial/projects;
+	export MODELS=../tutorial/models/;
+	make start

docker-components/Dockerfile_elasticsearch_phonetic

@@ -0,0 +1,5 @@
+FROM elasticsearch-oss:6.1.1
+ARG proxy
+ENV https_proxy $proxy
+RUN wget https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-phonetic/analysis-phonetic-6.1.1.zip -O /tmp/analysis-phonetic-6.1.1.zip `echo $proxy | sed 's/\(\S\S*\)/-e use_proxy=yes/'` > /tmp/analysis-phonetic-6.1.1.zip
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install file:///tmp/analysis-phonetic-6.1.1.zip

docker-components/Dockerfile_postgres_cstore

@@ -0,0 +1,21 @@
+FROM postgres:10.1
+ARG proxy
+ENV https_proxy $proxy
+
+RUN apt-get update -y -qq && \
+    apt-get -y -qq install protobuf-c-compiler libprotobuf-c0-dev unzip git build-essential
+
+RUN apt-get update \
+    && apt-get install -y \
+        postgresql-server-dev-$PG_MAJOR=$PG_VERSION \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src
+RUN git clone https://github.com/citusdata/cstore_fdw.git && \
+    cd cstore_fdw && \
+    make -j"$(nproc)" && \
+    make install && \
+    cd .. && \
+    rm -rf cstore_fdw
+
+WORKDIR /usr/src/postgres

docker-components/docker-compose-build-frontend.yml

@@ -0,0 +1,27 @@
+version: '2'
+
+# this docker configuation intends to build frontend
+# it is supposed that your declared some ENV variables:
+#   - ${FRONTEND} points to matchID-frontend (from GitHub, branch 'master')
+
+services:
+  build-frontend:
+    build:
+      context: ${FRONTEND}
+      dockerfile: Dockerfile.build
+      args:
+        proxy: ${http_proxy}
+    image: matchid-frontend:latest
+    container_name: matchid-build-front
+    volumes:
+      - ${FRONTEND}/index.html:/matchid/index.html
+      - ${FRONTEND}/.babelrc:/matchid/.babelrc
+      - ${FRONTEND}/.editorconfig:/matchid/.editorconfig
+      - ${FRONTEND}/.eslintignore:/matchid/.eslintignore
+      - ${FRONTEND}/.eslintrc.js:/matchid/.eslintrc.js
+      - ${FRONTEND}/matchIdConfig:/matchid/matchIdConfig/
+      - ${FRONTEND}/config:/matchid/config/
+      - ${FRONTEND}/src:/matchid/src/
+      - ${FRONTEND}/build:/matchid/build/
+      - ${FRONTEND}/static:/matchid/static/
+      - ${FRONTEND}/dist:/matchid/dist/

docker-components/docker-compose-dev-frontend.yml

@@ -0,0 +1,57 @@
+version: '2'
+
+# this docker configuation is for dev use only
+# it is supposed that your declared some ENV variables:
+#   - ${FRONTEND} points to matchID-frontend (from GitHub, branch 'master')
+#   - ${PROJECTS} points to your projects path
+#   - ${UPLOAD} points to your upload path
+# the dev conf allows you to change both python and nodejs code and conf live they will be
+# updated within the docker instances
+
+# for dev use it is recommanded to have at least 8Gb of memory and SSD and 4 vCPU, to ensure
+# good performance and testing.
+
+# for production use many adaptation will be needed if you use docker
+# - assume you docker images with a custum managed repo
+# - securize nginx routes allowing only stricly needed routes of API
+# - build the static files from node frontend and put them to the nginx
+# - build a cluster for elasticsearch
+# - use real volumes : S3-like storage for configurations (using volumes) of front and back,
+#   and DB-storage for ES volume like SSD volumes (or memory storage still better)
+
+services:
+  nginx-dev:
+    image: nginx
+    container_name: matchid-nginx-dev
+    depends_on:
+      - frontend-dev
+    volumes:
+      - ${BACKEND}/nginx/nginx-dev.conf:/etc/nginx/conf.d/default.conf
+    ports:
+      - "80:80"
+    restart: always
+
+  frontend-dev:
+    build:
+      context: ${FRONTEND}
+      dockerfile: Dockerfile.dev
+      args:
+        proxy: ${http_proxy}
+    image: matchid-frontend-dev:latest
+    container_name: matchid-frontend-dev
+    volumes:
+      - ${FRONTEND}/index.html:/matchid/index.html
+      - ${FRONTEND}/.babelrc:/matchid/.babelrc
+      - ${FRONTEND}/.editorconfig:/matchid/.editorconfig
+      - ${FRONTEND}/.eslintignore:/matchid/.eslintignore
+      - ${FRONTEND}/.eslintrc.js:/matchid/.eslintrc.js
+      - ${FRONTEND}/matchIdConfig:/matchid/matchIdConfig/
+      - ${FRONTEND}/config:/matchid/config/
+      - ${FRONTEND}/src:/matchid/src/
+      - ${FRONTEND}/build:/matchid/build/
+
+networks:
+  default:
+    external:
+      name: matchid
+