docker-compose.no.datahub.yml

version: '3.9'
services:

  minio:
    hostname: minio
    image: 'minio/minio'
    container_name: minio
    ports:
      - '9000:9000'
      - '9001:9001'
    volumes:
      - ./data/minio:/data
    environment:
      - MINIO_ROOT_USER=minio
      - MINIO_ROOT_PASSWORD=minio123
    command: server /data --console-address ":9001"
    networks:
      - ndsnet
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 5

  mc:
    image: minio/mc
    container_name: mc
    hostname: mc
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    networks:
      - ndsnet
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 minio minio123) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb minio/warehouse;
      /usr/bin/mc policy set public minio/warehouse;
      exit 0;
      "    
    depends_on:
      minio:
        condition: service_healthy  

  postgres:
    image: postgres
    hostname: postgres
    container_name: postgres
    environment:
      - POSTGRES_PASSWORD=ngods
      - POSTGRES_USER=ngods
      - POSTGRES_DB=ngods
    ports:
      - '5432:5432'
    networks:
      - ndsnet
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ngods"]
      interval: 5s
      timeout: 5s
      retries: 5
  
  aio:
    build: aio/
    container_name: aio
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    volumes:
      - ./data/html:/var/lib/ngods/html
      - ./data/stage:/var/lib/ngods/stage
      - ./data/spark/notebooks:/var/lib/ngods/notebooks
      - ./projects/dagster:/var/lib/ngods/dagster
      - ./projects/dbt:/var/lib/ngods/dbt
    ports:
      - '8888:8888' # Spark notebook port
      - '7077:7077' # Spark port
      - '8061:8061' # Spark master web ui port
      - '8062:8062' # Spark worker web ui port
      - '10000:10000' # Thrift ODBC/JDBC port
      - '10009:10009' # Kyuubi JDBC port
      - '18080:18080' # Spark history web ui port
      - '3070:3070' # Dagster web ui port
      - '9083:9083' # Metastore thrift
    networks:
      ndsnet:
        aliases:
          - spark
          - dagster
          - dbt
          - metastore
          - aio
    command: 
      - notebook
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
  
  trino:
    build: trino/
    container_name: trino
    volumes:
      - ./data/stage:/var/lib/ngods/stage
      - ./certs/trino.pass:/trino.pass
      - ./certs/trino_key_and_cert.pem:/trino_key_and_cert.pem
      - ./certs/trino_key.pem:/trino_key.pem
      - ./certs/trino_cert.pem:/trino_cert.pem
    ports:
      - '8060:8060' # Trino
      - '8063:8063' # Trino
    networks:
      ndsnet:
        aliases:
          - trino
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000

        
  cube:
    image: cubejs/cube
    container_name: cube
    hostname: cube
    volumes:
      - ./conf/cube:/cube/conf
    ports:
      - 3000:3000 
      - 4000:4000
      - 3245:3245
    networks:
      - ndsnet
    depends_on:
      - aio

  metabase:
    container_name: metabase
    depends_on:
      - cube
    hostname: metabase
    build: metabase/
    volumes:
      - ./conf/metabase:/conf
    ports:
      - 3030:3030
    networks:
      - ndsnet
  
networks:
  ndsnet:
    driver: bridge