docker-compose.yml

version: '3' 
services:
  zookeeper:
    image: fjardim/mds-kafka-zookeeper
    hostname: zookeeper
    container_name: zookeeper
    ports:
       - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  kafka-broker:
    image: fjardim/mds-kafka-broker
    container_name: kafka-broker
    hostname: kafka-broker
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:9092
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: kafka-broker
      #CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:9092
      #CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'false'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
      # AUTO_REGISTER_SCHEMAS : 'false'

  kafka-schema-registry:
    image: fjardim/mds-kafka-schema-registry
    hostname: kafka-schema-registry
    container_name: kafka-schema-registry
    depends_on:
      - kafka-broker
    ports:
      - "8071:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081

  kafka-connect:
    image: fjardim/mds-kafka-connect
    hostname: kafka-connect
    container_name: kafka-connect
    depends_on:
      - kafka-broker
      - kafka-schema-registry
      - postgres
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081
      # CLASSPATH required due to CC-2422
      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.1.0.jar
      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
      
         
  kafka-control-center:
    image: fjardim/mds-kafka-control-center
    hostname: kafka-control-center
    container_name: kafka-control-center
    depends_on:
      - kafka-broker
      - kafka-schema-registry
      - kafka-connect
      - kafka-ksqldb-server
      - kafka-ksqldb-cli
      - postgres
    ports:
      - "9021:9021"
    environment:
      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
      CONTROL_CENTER_CONNECT_CLUSTER: 'kafka-connect:8083'
      CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'kafka-connect:8083'
      CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://kafka-ksqldb-server:8088"
      CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://kafka-ksqldb-server:8088"
      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://kafka-schema-registry:8081"
      CONTROL_CENTER_REPLICATION_FACTOR: 1
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
      CONTROL_CENTER_CONNECT_HEALTHCHECK_ENDPOINT: '/connectors'
      PORT: 9021

  kafka-ksqldb-server:
    image: fjardim/mds-kafka-ksqldb-server
    hostname: kafka-ksqldb-server
    container_name: kafka-ksqldb-server
    depends_on:
      - kafka-broker
      - kafka-connect
    ports:
      - "8068:8088"
    environment:
      KSQL_CONFIG_DIR: "/etc/ksql"
      KSQL_BOOTSTRAP_SERVERS: "kafka-broker:9092"
      KSQL_HOST_NAME: ksqldb-server
      KSQL_LISTENERS: "http://0.0.0.0:8088"
      KSQL_CACHE_MAX_BYTES_BUFFERING: 0
      KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://kafka-schema-registry:8081"
      KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
      KSQL_KSQL_CONNECT_URL: "http://kafka-connect:8083"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true'
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true'

  kafka-ksqldb-cli:
    image: fjardim/mds-kafka-sqldb-cli
    container_name: kafka-ksqldb-cli
    hostname: kafka-ksqldb-cli
    depends_on:
      - kafka-broker
      - kafka-connect
      - kafka-ksqldb-server
    entrypoint: /bin/sh
    tty: true

  
  kafka-rest-proxy:
   image: fjardim/mds-kafka-rest
   hostname: kafka-rest-proxy
   depends_on:
     - kafka-broker
     - kafka-schema-registry
   ports:
     - 8082:8082
   container_name: kafka-rest-proxy
   environment:
     KAFKA_REST_HOST_NAME: rest-proxy
     KAFKA_REST_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
     KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
     KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'


  nifi:
    image: fjardim/mds-nifi
    container_name: nifi
    hostname: nifi
    volumes:
      - ./nifi/flow:/opt/nifi/nifi-current/flowfile_repository
      - ./nifi/util:/util
      - ./scripts:/scripts
    environment:
      NIFI_WEB_HTTP_PORT: "9090"
      SINGLE_USER_CREDENTIALS_USERNAME: "admin"
      SINGLE_USER_CREDENTIALS_PASSWORD: "admin"
    ports:
      - "9090:9090"
      - "8443:8443"
    deploy:
      resources:
        limits:
          memory: 2048m

  postgres:
    image: fjardim/mds-postgres
    container_name: postgres
    hostname: postgres
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      POSTGRES_DB: admin 
    ports:
      - 5442:5432
    volumes:
      - ./postgres/postgres-db-volume:/var/lib/postgresql/data
      - ./postgres:/util
      - ./scripts:/scripts
    #healthcheck:
    #  test: ["CMD", "pg_isready"]
    #  interval: 5s
    #  retries: 5
  
  
  spark-master:
    image: fjardim/mds-spark
    hostname: spark-master
    container_name: spark-master
    command: 
      - /bin/sh
      - -c
      - |
        /usr/local/spark/sbin/start-master.sh
        start-notebook.sh --NotebookApp.token=''
    #environment:
    #  PYSPARK_SUBMIT_ARGS: "--packages io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.392"
    #env_file:
    #  - ./data/jupyter/jupyter.env
    #working_dir: /home/root
    #entrypoint: ["chmod", "+x", "/env/start-master.sh","/env","./start-master.sh"]
    ports:
      - 8889:8888
      - 4040:4040
      - 4041:4041
      - 4042:4042
      - 4043:4043
      - 8180:8080
      - 7077:7077
    volumes:
      - ./spark/util:/util/
      - ./spark/work:/home/user 
      - ./spark/env:/env 
      - ./scripts:/scripts
      #- ./jupyter/conf:/usr/local/spark/conf
    #environment:
    #  - GRANT_SUDO=yes
    #  - CHOWN_HOME=yes
    #  - NB_USER=spark
    #  - NB_GID=100
    #  - CHOWN_HOME_OPTS='-R'
      #- GEN_CERT=yes

    #   SPARK_MASTER: local[*]
    #   JUPYTER_PORT: 8889
    #deploy:
    #  resources:
    #    limits:
    #      memory: 500m

  spark-worker:
    image: fjardim/mds-spark
    hostname: spark-worker
    container_name: spark-worker
    #command: start-notebook.sh --NotebookApp.token='' 
    command: 
      - /bin/sh
      - -c
      - |
        /usr/local/spark/sbin/start-worker.sh spark-master:7077
        start-notebook.sh --NotebookApp.token='' 
    #command: /usr/local/spark/sbin/start-worker.sh jupyter-spark:7077
    #environment:
    #  PYSPARK_SUBMIT_ARGS: "--packages io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.392"
    env_file:
      - ./spark/env/jupyter.env
    #working_dir: /home/root
    ports:
      - 5040:4040
      - 5041:4041
      - 5042:4042
      - 5043:4043
      - 8881:8081
      - 36533:36533
    volumes:
      - ./spark/util:/util/
      - ./spark/work:/home/user 
      - ./scripts:/scripts
    environment:
      SPARK_MASTER: spark-master
    #   JUPYTER_PORT: 8889
    depends_on:
        - spark-master
    #deploy:
    #  resources:
    #    limits:
    #      memory: 1g

  metabase:
    image: fjardim/mds-metabase
    container_name: metabase
    hostname: metabase
    ports:
      - "3000:3000"
    depends_on:
      - postgres
    environment:
      MB_DB_TYPE: postgres
      MB_DB_DBNAME: metabase
      MB_DB_PORT: 5432
      MB_DB_USER: admin
      MB_DB_PASS: admin
      MB_DB_HOST: postgres
      MB_PASSWORD_COMPLEXITY: "weak"
      MB_PASSWORD_LENGTH: 4

  elasticsearch:
    image: fjardim/mds-elasticsearch
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      discovery.type: "single-node"
      ES_JAVA_OPTS: "-Xms2g -Xmx2g"
      xpack.monitoring.enabled: "true"
    volumes:
      - ./elastic/esdata:/usr/share/elasticsearch/data
      - ./scripts:/scripts
      
  kibana:
    image: fjardim/mds-kibana
    ports:
      - "5601:5601"
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200
    depends_on:
      - elasticsearch


  trino:
    image: fjardim/mds-trino 
    #image: trinodb/trino:latest
    container_name: trino
    hostname: trino
    ports: 
      - "8080:8080" 
    depends_on:
      - postgres
      #- metastore
    volumes: 
      - ./trino/catalog:/etc/trino/catalog  
      - ./trino/conf:/conf
      - ./scripts:/scripts

  hive:
    hostname: hive
    #image: apache/hive:4.0.0-alpha-2
    image: fjardim/mds-hive
    container_name: hive
    environment:
      AWS_ACCESS_KEY_ID: datalake
      AWS_SECRET_ACCESS_KEY: datalake
      HIVE_CUSTOM_CONF_DIR: "/hive_custom_conf"
      SERVICE_NAME: hiveserver2
      SERVICE_OPTS: "-Dhive.metastore.uris=thrift://metastore:9083" 
      IS_RESUME: "true"
      #HIVE_VERSION: "3.1.3"
    ports:
       - "10000:10000"
       - "10002:10002"
    depends_on:
        - hive-metastore
    user: root
    volumes:
       - ./hive/conf:/hive_custom_conf
       - ./hive:/util

  hive-metastore:
    hostname: metastore
    image: fjardim/mds-hive-metastore
    #image: apache/hive:4.0.0-alpha-2
    container_name: metastore
    environment:
      AWS_ACCESS_KEY_ID: datalake
      AWS_SECRET_ACCESS_KEY: datalake
      HIVE_CUSTOM_CONF_DIR: "/hive_custom_conf"
      SERVICE_NAME: metastore
      #SERVICE_OPTS: "-Dhive.metastore.uris=thrift://metastore:9083" 
      IS_RESUME: "true"
      DB_DRIVER: postgres 
      SERVICE_OPTS: "-Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore_db -Djavax.jdo.option.ConnectionUserName=admin -Djavax.jdo.option.ConnectionPassword=admin" 
    ports:
       - "9083:9083"
    depends_on: 
      - postgres
    user: root
    volumes:
       - ./hive/meta:/opt/hive/data/warehouse 
       - ./hive/conf:/hive_custom_conf
       - ./hive:/util

  minio:
    image: fjardim/mds-minio
    container_name: minio
    command:  server --console-address ":9001" /data
    ports:
      - "9050:9000"
      - "9051:9001"
    hostname: minio
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: minioadmin
      MINIO_ACCESS_KEY: datalake
      MINIO_SECRET_KEY: datalake
    volumes:
      - ./minio/data1:/data
      - ./scripts:/scripts

  superset:
    env_file: ./superset/docker/env-non-dev
    image: fjardim/mds-superset
    container_name: superset_app
    hostname: superset_app
    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
    user: "root"
    restart: unless-stopped
    ports:
      - 8088:8088
    depends_on: 
      - postgres
      #- redis
      - trino
    volumes: 
      - ./superset/docker:/app/docker
      - ./superset/superset_home:/app/superset_home
      - ./scripts:/scripts


  # pinot-zookeeper:
  #   image: fjardim/mds-pinot-zookeeper
  #   container_name: pinot-zookeeper
  #   ports:
  #     - "2182:2181"
  #   environment:
  #     ZOOKEEPER_CLIENT_PORT: 2181
  #     ZOOKEEPER_TICK_TIME: 2000

  # pinot-controller:
  #   image: fjardim/mds-pinot
  #   command: "StartController -zkAddress pinot-zookeeper:2181"
  #   container_name: pinot-controller
  #   restart: unless-stopped
  #   ports:
  #     - "9000:9000"
  #   environment:
  #     JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"
  #   depends_on:
  #     - pinot-zookeeper

  # pinot-broker:
  #   image: fjardim/mds-pinot
  #   command: "StartBroker -zkAddress pinot-zookeeper:2181"
  #   restart: unless-stopped
  #   container_name: "pinot-broker"
  #   ports:
  #     - "8099:8099"
  #   environment:
  #     JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"
  #   depends_on:
  #     - pinot-controller

  # pinot-server:
  #   image: fjardim/mds-pinot
  #   command: "StartServer -zkAddress pinot-zookeeper:2181"
  #   restart: unless-stopped
  #   container_name: "pinot-server"
  #   ports:
  #     - "8098:8098"
  #   environment:
  #     JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
  #   depends_on:
  #     - pinot-broker
  #     - kafka-broker

  namenode:
    image: fjardim/mds-namenode
    container_name: namenode
    hostname: namenode
    volumes:
      - ./hadoop/hdfs/namenode:/hadoop/dfs/name
      - ./hadoop/util:/util
    env_file:
      - ./hadoop/hadoop.env
    ports:
      - "9870:9870"
    deploy:
      resources:
        limits:
          memory: 500m
  
  datanode:
    image: fjardim/mds-datanode
    container_name: datanode
    hostname: datanode
    volumes:
      - ./hadoop/hdfs/datanode:/hadoop/dfs/data
      - ./hadoop/util:/util
    env_file:
      - ./hadoop/hadoop.env
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    depends_on:
      - namenode
    ports:
      - "9864:9864"
    deploy:
      resources:
        limits:
          memory: 500m

  hue:
    image: fjardim/mds-hue
    hostname: hue
    container_name: hue
    dns: 8.8.8.8
    ports:
    - "8888:8888"
    volumes:
      - ./hue/hue.ini:/usr/share/hue/desktop/conf/z-hue.ini
    depends_on:
      - "postgres"
    deploy:
      resources:
        limits:
          memory: 500m