docker-compose.yml

# docker-compose supports environment variable substitution with the ${VARIABLE-NAME} syntax.
# Environment variables can be sourced in a variety of ways.  One of those ways is through
# a well known '.env' file located in the same folder as the docker-compose.yml file.  See the Docker
# documentation for details: https://docs.docker.com/compose/environment-variables/#the-env-file
# 
# This feature is being used to parameterize some values within this file.  In this directory is also
# a .env file, which is actually a symbolic link to <examples-root>/utils/config.env.  That file
# contains values which get substituted here when docker-compose parses this file.
#
# If you'd like to view the docker-compose.yml file rendered with its environment variable substituions
# you can execute the `docker-compose config` command.  Take note that some demos provide additional 
# environment variable values by exporting them in a script prior to running `docker-compose up`.
---
version: '2'
services:
  zookeeper:
    container_name: zookeeper
    image: ${REPOSITORY}/cp-zookeeper:${CONFLUENT_DOCKER_TAG}
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  kafka:
    container_name: kafka
    image: ${REPOSITORY}/cp-server:${CONFLUENT_DOCKER_TAG}
    depends_on:
      - zookeeper
    #ports:
    # This would expose 9092 for external connections to the broker
    # Use kafka:29092 for connections internal on the docker network
    # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
    #  - 9092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'true'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'

  schema-registry:
    container_name: schema-registry
    image: ${REPOSITORY}/cp-schema-registry:${CONFLUENT_DOCKER_TAG}
    depends_on:
      - kafka
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092

  ksqldb-server:
    container_name: ksqldb-server
    image: ${REPOSITORY}/cp-ksqldb-server:${CONFLUENT_DOCKER_TAG}
    ports: 
      - 8088:8088
      - 8083:8083
    depends_on:
      - kafka
      - schema-registry
    environment:
      KSQL_CUB_KAFKA_TIMEOUT: 300
      KSQL_BOOTSTRAP_SERVERS: kafka:29092
      KSQL_LISTENERS: http://0.0.0.0:8088
      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      KSQL_AUTO_OFFSET_RESET: "earliest"
      KSQL_COMMIT_INTERVAL_MS: 2000
      KSQL_KSQL_HIDDEN_TOPICS: '^_.*'

      # ksqlDB embedded Connect worker
      KSQL_CONNECT_BOOTSTRAP_SERVERS: kafka:29092
      KSQL_CONNECT_REST_ADVERTISED_HOST_NAME: ksqldb
      KSQL_CONNECT_GROUP_ID: ksqldb-kafka-connect-group-01
      KSQL_CONNECT_CONFIG_STORAGE_TOPIC: _ksqldb-kafka-connect-group-01-configs
      KSQL_CONNECT_OFFSET_STORAGE_TOPIC: _ksqldb-kafka-connect-group-01-offsets
      KSQL_CONNECT_STATUS_STORAGE_TOPIC: _ksqldb-kafka-connect-group-01-status
      KSQL_CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      KSQL_CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      KSQL_CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      KSQL_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      KSQL_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      KSQL_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      KSQL_CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: '[%d] %p %X{connector.context}%m (%c:%L)%n'
      KSQL_CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/,/data/connect-jars'

      KSQL_KSQL_SERVICE_ID: "ksql-cluster"
      CONNECT_HOST: ksqldb-server
      ELASTIC_HOST: elasticsearch
      GRAFANA_HOST: grafana
    # If you want to use the Confluent Hub installer to d/l component, but make them available
    # when running this offline, spin up the stack once and then run : 
    #   docker cp ksqldb:/usr/share/confluent-hub-components ./data/connect-jars
    volumes:
      - $PWD/data:/data
      - $PWD/ksql/ksql-clickstream-demo/demo/:/scripts
      - $PWD/ksql/ksql-clickstream-demo/demo/connect-config:/usr/share/java/null-smt
      - $PWD/confluent-hub-components/:/usr/share/confluent-hub-components

  ksqldb-cli:
    container_name: ksqldb-cli
    image: ${REPOSITORY}/cp-ksqldb-cli:${CONFLUENT_DOCKER_TAG}
    depends_on:
      - ksqldb-server
    entrypoint: /bin/sh
    tty: true
    volumes:
      - $PWD/ksql/ksql-clickstream-demo/demo/:/scripts

  tools:
    image: cnfltraining/training-tools:5.4
    hostname: tools
    container_name: tools
    entrypoint: /bin/bash
    tty: true

  elasticsearch:
    container_name: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.0
    ports:
      - 9200:9200
    environment:
      xpack.security.enabled: "false"
      ES_JAVA_OPTS: "-Xms1g -Xmx1g"
      ELASTIC_HOST: elasticsearch
    volumes:
     - $PWD/ksql/ksql-clickstream-demo/demo:/scripts

  grafana:
    container_name: grafana
    image: grafana/grafana:5.2.4
    ports:
      - 3000:3000
    environment:
      GRAFANA_HOST: grafana
      GF_SECURITY_ADMIN_USER: user
      GF_SECURITY_ADMIN_PASSWORD: user
    volumes:
     - $PWD/ksql/ksql-clickstream-demo/demo:/scripts

  control-center:
    image: ${REPOSITORY}/cp-enterprise-control-center:${CONFLUENT_DOCKER_TAG}
    hostname: control-center
    container_name: control-center
    depends_on:
      - kafka
      - schema-registry
      - ksqldb-server
    ports:
      - 9021:9021
    environment:
      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka:29092'
      CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'ksqldb-server:8083'
      CONTROL_CENTER_KSQL_CLICKSTREAM_URL: "http://ksqldb-server:8088"
      CONTROL_CENTER_KSQL_CLICKSTREAM_ADVERTISED_URL: "http://localhost:8088"
      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
      CONTROL_CENTER_REPLICATION_FACTOR: 1
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
      PORT: 9021