docker-compose.yml

version: '3.8'
services:

  # Apache Flink Components:

  sql-client:
    container_name: sql-client
    build:
      context: sql-client
      dockerfile: Dockerfile
    depends_on:
      - broker
      - jobmanager
    environment:
      FLINK_JOBMANAGER_HOST: jobmanager
      KAFKA_BOOTSTRAP: broker

  jobmanager:
    image: flink:${FLINK_TAG}
    container_name: jobmanager
    build:
      context: jobmanager
      dockerfile: Dockerfile
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager

  taskmanager:
    image: flink:${FLINK_TAG}
    container_name: taskmanager
    depends_on:
      - jobmanager
    command: taskmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 10

  # Confluent Platform Components below:

  broker:
    container_name: broker
    hostname: broker
    image: confluentinc/cp-server:${CP_TAG}
    ports:
      - "9101:9101"
      - "9092:9092"
      - "8090:8090"
    restart: always
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
      KAFKA_DEFAULT_MIN_ISR: 1
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_CLUSTER_LINK_METADATA_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: 'false'
      KAFKA_CONFLUENT_BALANCER_ENABLE: 'false'
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_CONFLUENT_HTTP_SERVER_LISTENERS: "http://0.0.0.0:8090"
      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: "http://schemaregistry:8084"
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
      KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      CLUSTER_ID: 'ZWe3nnZwTrKSM0aM2doAxQ'

      # Metrics Reporting
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092

      # Override Replication Factors for internal topics
      KAFKA_CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      KAFKA_CONFLUENT_METRICS_REPORTER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_METRICS_ENABLE: 'true'

  control-center:
    container_name: control-center
    hostname: control-center
    image: confluentinc/cp-enterprise-control-center:${CP_TAG}
    depends_on:
      - broker
      - schemaregistry
    ports:
      - "9021:9021"
    restart: always
    environment:
      CONTROL_CENTER_BOOTSTRAP_SERVERS: broker:29092
      CONTROL_CENTER_SCHEMA_REGISTRY_URL: http://schemaregistry:8084
      CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: connect:8083
      CONTROL_CENTER_STREAMS_NUM_STREAM_THREADS: "5"
      CONTROL_CENTER_REPLICATION_FACTOR: 1
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
      PORT: 9021

  restproxy:
    container_name: restproxy
    hostname: restproxy
    image: confluentinc/cp-kafka-rest:${CP_TAG}
    depends_on:
      - broker
      - schemaregistry
    ports: 
      - 8082:8082
    restart: always
    environment:
      KAFKA_REST_BOOTSTRAP_SERVERS: broker:29092
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://schemaregistry:8084

  schemaregistry:
    container_name: schemaregistry
    hostname: schemaregistry
    image: confluentinc/cp-schema-registry:${CP_TAG}
    depends_on:
      - broker
    ports:
      - '8084:8084'
    restart: always
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker:29092
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8084

  connect:
    container_name: connect
    hostname: connect
    image: confluentinc/cp-enterprise-replicator:${CP_TAG}
    ports:
      - '8083:8083'
    depends_on:
      - broker
      - schemaregistry
    restart: always
    environment:
      CONNECT_BOOTSTRAP_SERVERS: broker:29092
      CONNECT_LISTENERS: http://connect:8083
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "source"
      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_BOOTSTRAP_SERVERS: broker:29092
      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
      CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_BOOTSTRAP_SERVERS: broker:29092
      CONNECT_CONFLUENT_METADATA_BOOTSTRAP_SERVER_URLS: http://broker:8090
      # Connect Custom Topics
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry:8084
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry:8084
      CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: All
      CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/connect-jars
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
    command:
      - bash
      - -c
      - |
        echo "Installing Connector"
        confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.6.3
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run &
        #
        sleep infinity