Created
April 23, 2023 09:24
-
-
Save shivaluma/c673a48907a6454fe969639229b328ee to your computer and use it in GitHub Desktop.
Kafka cluster using conduktor with kafka connect and ksqldb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| version: '2.1' | |
| services: | |
| zoo1: | |
| image: confluentinc/cp-zookeeper:7.3.2 | |
| hostname: zoo1 | |
| container_name: zoo1 | |
| ports: | |
| - "2181:2181" | |
| environment: | |
| ZOOKEEPER_CLIENT_PORT: 2181 | |
| ZOOKEEPER_SERVER_ID: 1 | |
| ZOOKEEPER_SERVERS: zoo1:2888:3888 | |
| kafka1: | |
| image: confluentinc/cp-kafka:7.3.2 | |
| hostname: kafka1 | |
| container_name: kafka1 | |
| ports: | |
| - "9092:9092" | |
| - "29092:29092" | |
| - "9999:9999" | |
| environment: | |
| KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-185.223.207.15}:9092,DOCKER://host.docker.internal:29092 | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT | |
| KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL | |
| KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" | |
| KAFKA_BROKER_ID: 1 | |
| KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
| KAFKA_JMX_PORT: 9001 | |
| KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-185.223.207.15} | |
| KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer | |
| KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" | |
| depends_on: | |
| - zoo1 | |
| kafka2: | |
| image: confluentinc/cp-kafka:7.3.2 | |
| hostname: kafka2 | |
| container_name: kafka2 | |
| ports: | |
| - "9093:9093" | |
| environment: | |
| KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-185.223.207.15}:9093,DOCKER://host.docker.internal:29093 | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT | |
| KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL | |
| KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" | |
| KAFKA_BROKER_ID: 2 | |
| KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
| KAFKA_JMX_PORT: 9001 | |
| KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-185.223.207.15} | |
| KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer | |
| KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" | |
| depends_on: | |
| - zoo1 | |
| kafka3: | |
| image: confluentinc/cp-kafka:7.3.2 | |
| hostname: kafka3 | |
| container_name: kafka3 | |
| ports: | |
| - "9094:9094" | |
| environment: | |
| KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-185.223.207.15}:9094,DOCKER://host.docker.internal:29094 | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT | |
| KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL | |
| KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" | |
| KAFKA_BROKER_ID: 3 | |
| KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
| KAFKA_JMX_PORT: 9001 | |
| KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-185.223.207.15} | |
| KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer | |
| KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" | |
| depends_on: | |
| - zoo1 | |
| kafka-schema-registry: | |
| image: confluentinc/cp-schema-registry:7.3.2 | |
| hostname: kafka-schema-registry | |
| container_name: kafka-schema-registry | |
| ports: | |
| - "8081:8081" | |
| environment: | |
| SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 | |
| SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry | |
| SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 | |
| depends_on: | |
| - zoo1 | |
| - kafka1 | |
| kafka-rest-proxy: | |
| image: confluentinc/cp-kafka-rest:7.3.2 | |
| hostname: kafka-rest-proxy | |
| container_name: kafka-rest-proxy | |
| ports: | |
| - "8082:8082" | |
| environment: | |
| # KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181 | |
| KAFKA_REST_LISTENERS: http://0.0.0.0:8082/ | |
| KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/ | |
| KAFKA_REST_HOST_NAME: kafka-rest-proxy | |
| KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 | |
| depends_on: | |
| - zoo1 | |
| - kafka1 | |
| - kafka-schema-registry | |
| kafka-connect: | |
| image: confluentinc/cp-kafka-connect:7.3.2 | |
| hostname: kafka-connect | |
| container_name: kafka-connect | |
| ports: | |
| - "8083:8083" | |
| environment: | |
| CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092" | |
| CONNECT_REST_PORT: 8083 | |
| CONNECT_GROUP_ID: compose-connect-group | |
| CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs | |
| CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets | |
| CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status | |
| CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter | |
| CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081' | |
| CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter | |
| CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081' | |
| CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
| CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
| CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" | |
| CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" | |
| CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" | |
| CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" | |
| CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" | |
| CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" | |
| CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars,/usr/share/confluent-hub-components' | |
| volumes: | |
| - ./connectors:/etc/kafka-connect/jars/ | |
| depends_on: | |
| - zoo1 | |
| - kafka1 | |
| - kafka-schema-registry | |
| - kafka-rest-proxy | |
| command: | |
| - bash | |
| - -c | |
| - | | |
| confluent-hub install --no-prompt debezium/debezium-connector-mysql:latest | |
| confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.4.0 | |
| /etc/confluent/docker/run | |
| ksqldb-server: | |
| image: confluentinc/cp-ksqldb-server:7.3.2 | |
| hostname: ksqldb-server | |
| container_name: ksqldb-server | |
| ports: | |
| - "8088:8088" | |
| environment: | |
| KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 | |
| KSQL_LISTENERS: http://0.0.0.0:8088/ | |
| KSQL_KSQL_SERVICE_ID: ksqldb-server_ | |
| depends_on: | |
| - zoo1 | |
| - kafka1 | |
| conduktor-platform: | |
| image: conduktor/conduktor-platform:latest | |
| ports: | |
| - "8080:8080" | |
| volumes: | |
| - conduktor_data:/var/conduktor | |
| depends_on: | |
| - kafka1 | |
| - kafka-schema-registry | |
| - kafka-connect | |
| environment: | |
| CDK_ORGANIZATION_NAME: "demo" | |
| CDK_ADMIN_EMAIL: "admin@admin.io" | |
| CDK_ADMIN_PASSWORD: "admin" | |
| CDK_CLUSTERS_0_ID: "default" | |
| CDK_CLUSTERS_0_NAME: "My Local Kafka Cluster" | |
| CDK_CLUSTERS_0_COLOR: "#0013E7" | |
| CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "PLAINTEXT://kafka1:19092" | |
| CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081" | |
| CDK_CLUSTERS_0_KAFKACONNECTS_0_URL: "http://kafka-connect:8083" | |
| CDK_CLUSTERS_0_KAFKACONNECTS_0_NAME: "full stack kafka connect" | |
| volumes: | |
| conduktor_data: {} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment