From b2d0d209176d21b0227cb4c426c07ea521dfbbc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20P=C3=B3=C5=82grabia?= Date: Sat, 28 Dec 2024 23:27:48 +0100 Subject: [PATCH 1/2] Working setup for apache/kafka and zookeeper (no confluent). --- 2024/08/chat_demo1/tools/docker-compose.yaml | 14 +++++++---- .../chat_demo1/tools/docker-compose.yaml.bak | 21 ----------------- .../tools/docker-compose.yaml.bak.2 | 23 ------------------- ...ak.3 => docker-compose.yaml.confluent.old} | 6 +++-- 4 files changed, 13 insertions(+), 51 deletions(-) delete mode 100644 2024/08/chat_demo1/tools/docker-compose.yaml.bak delete mode 100644 2024/08/chat_demo1/tools/docker-compose.yaml.bak.2 rename 2024/08/chat_demo1/tools/{docker-compose.yaml.bak.3 => docker-compose.yaml.confluent.old} (75%) diff --git a/2024/08/chat_demo1/tools/docker-compose.yaml b/2024/08/chat_demo1/tools/docker-compose.yaml index e7c3a3f..7004639 100644 --- a/2024/08/chat_demo1/tools/docker-compose.yaml +++ b/2024/08/chat_demo1/tools/docker-compose.yaml @@ -1,24 +1,28 @@ services: zookeeper: - image: confluentinc/cp-zookeeper:7.4.8 + image: zookeeper:latest environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 + ALLOW_ANONYMOUS_LOGIN: true ports: - "22181:2181" broker: - image: confluentinc/cp-kafka:7.4.8 + image: apache/kafka:latest depends_on: - zookeeper environment: - KAFKA_BROKER_ID: 1 + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:29092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:29092,CONTROLLER://localhost:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 TOPIC_AUTO_CREATE: true + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 ports: - "29092:29092" diff --git a/2024/08/chat_demo1/tools/docker-compose.yaml.bak b/2024/08/chat_demo1/tools/docker-compose.yaml.bak deleted file mode 100644 index 2ba8c8c..0000000 --- a/2024/08/chat_demo1/tools/docker-compose.yaml.bak +++ /dev/null @@ -1,21 +0,0 @@ -services: - zookeeper: - image: confluentinc/cp-zookeeper:latest - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - ports: - - '2181:2181' - broker: - image: confluentinc/cp-kafka:latest - depends_on: - - zookeeper - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - ports: - - "29092:29092" diff --git a/2024/08/chat_demo1/tools/docker-compose.yaml.bak.2 b/2024/08/chat_demo1/tools/docker-compose.yaml.bak.2 deleted file mode 100644 index 4c76793..0000000 --- a/2024/08/chat_demo1/tools/docker-compose.yaml.bak.2 +++ /dev/null @@ -1,23 +0,0 @@ -version: '2' -services: - zookeeper: - image: confluentinc/cp-zookeeper:7.4.4 - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - ports: - - 22181:2181 - - kafka: - image: confluentinc/cp-kafka:7.4.4 - depends_on: - - zookeeper - ports: - - 29092:29092 - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 diff --git a/2024/08/chat_demo1/tools/docker-compose.yaml.bak.3 b/2024/08/chat_demo1/tools/docker-compose.yaml.confluent.old similarity index 75% rename from 2024/08/chat_demo1/tools/docker-compose.yaml.bak.3 rename to 2024/08/chat_demo1/tools/docker-compose.yaml.confluent.old index 1ab0253..e7c3a3f 100644 --- a/2024/08/chat_demo1/tools/docker-compose.yaml.bak.3 +++ b/2024/08/chat_demo1/tools/docker-compose.yaml.confluent.old @@ -1,22 +1,24 @@ services: zookeeper: - image: confluentinc/cp-zookeeper:7.4.4 + image: confluentinc/cp-zookeeper:7.4.8 environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 ports: - "22181:2181" broker: - image: confluentinc/cp-kafka:7.4.4 + image: confluentinc/cp-kafka:7.4.8 depends_on: - zookeeper environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:29092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + TOPIC_AUTO_CREATE: true ports: - "29092:29092" From a230667e74d4d1b7b10b61aa7b624b19309410d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20P=C3=B3=C5=82grabia?= Date: Sat, 28 Dec 2024 23:31:09 +0100 Subject: [PATCH 2/2] Small TODO marked in README. --- 2024/08/chat_demo1/tools/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/2024/08/chat_demo1/tools/README.md b/2024/08/chat_demo1/tools/README.md index 2e42fe5..a2ccf48 100644 --- a/2024/08/chat_demo1/tools/README.md +++ b/2024/08/chat_demo1/tools/README.md @@ -4,3 +4,6 @@ Seems like there is a behavior change between 7.4.4 and latest (7.8.0, too) imag 28.12.2024 - last 7.4.x tag 7.4.8 version seems to be working well. Needs to be checked what's the difference between 7.4.8 and 7.5.0 + +28.12.2024 - migrated to apache/kafka:latest and zookeeper:latest but kafka version needs to be synchronized with the client ones. + Switch it to 3.9.0 at least.