Skip to content

Commit

Permalink
kafka: switch to apache/kafka container (#28077)
Browse files Browse the repository at this point in the history
  • Loading branch information
mshima authored Dec 4, 2024
1 parent 8264ff0 commit 03d45f9
Show file tree
Hide file tree
Showing 14 changed files with 57 additions and 209 deletions.
18 changes: 3 additions & 15 deletions generators/app/__snapshots__/generator.spec.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -329,12 +329,11 @@ exports[`generator - app with default config should match snapshot 1`] = `
"devServerPort": 4200,
"devServerPortProxy": 9000,
"dockerContainers": {
"apache/kafka-native": "apache-kafka-native-placeholder",
"apachepulsar/pulsar": "apachepulsar-pulsar-placeholder",
"cassandra": "cassandra-placeholder",
"cassandraImage": "cassandra-image-placeholder",
"cassandraTag": "cassandra-tag-placeholder",
"confluentinc/cp-kafka": "confluentinc-cp-kafka-placeholder",
"confluentinc/cp-zookeeper": "confluentinc-cp-zookeeper-placeholder",
"consul": "consul-placeholder",
"consulConfigLoader": "consul-config-loader-placeholder",
"consulConfigLoaderImage": "consul-config-loader-image-placeholder",
Expand Down Expand Up @@ -449,9 +448,6 @@ exports[`generator - app with default config should match snapshot 1`] = `
"zipkin": "zipkin-placeholder",
"zipkinImage": "zipkin-image-placeholder",
"zipkinTag": "zipkin-tag-placeholder",
"zookeeper": "zookeeper-placeholder",
"zookeeperImage": "zookeeper-image-placeholder",
"zookeeperTag": "zookeeper-tag-placeholder",
},
"dockerServices": [
"app",
Expand Down Expand Up @@ -999,12 +995,11 @@ exports[`generator - app with gateway should match snapshot 1`] = `
"devServerPort": 4200,
"devServerPortProxy": 9000,
"dockerContainers": {
"apache/kafka-native": "apache-kafka-native-placeholder",
"apachepulsar/pulsar": "apachepulsar-pulsar-placeholder",
"cassandra": "cassandra-placeholder",
"cassandraImage": "cassandra-image-placeholder",
"cassandraTag": "cassandra-tag-placeholder",
"confluentinc/cp-kafka": "confluentinc-cp-kafka-placeholder",
"confluentinc/cp-zookeeper": "confluentinc-cp-zookeeper-placeholder",
"consul": "consul-placeholder",
"consulConfigLoader": "consul-config-loader-placeholder",
"consulConfigLoaderImage": "consul-config-loader-image-placeholder",
Expand Down Expand Up @@ -1119,9 +1114,6 @@ exports[`generator - app with gateway should match snapshot 1`] = `
"zipkin": "zipkin-placeholder",
"zipkinImage": "zipkin-image-placeholder",
"zipkinTag": "zipkin-tag-placeholder",
"zookeeper": "zookeeper-placeholder",
"zookeeperImage": "zookeeper-image-placeholder",
"zookeeperTag": "zookeeper-tag-placeholder",
},
"dockerServices": [
"app",
Expand Down Expand Up @@ -1664,12 +1656,11 @@ exports[`generator - app with microservice should match snapshot 1`] = `
"devServerPort": undefined,
"devServerPortProxy": undefined,
"dockerContainers": {
"apache/kafka-native": "apache-kafka-native-placeholder",
"apachepulsar/pulsar": "apachepulsar-pulsar-placeholder",
"cassandra": "cassandra-placeholder",
"cassandraImage": "cassandra-image-placeholder",
"cassandraTag": "cassandra-tag-placeholder",
"confluentinc/cp-kafka": "confluentinc-cp-kafka-placeholder",
"confluentinc/cp-zookeeper": "confluentinc-cp-zookeeper-placeholder",
"consul": "consul-placeholder",
"consulConfigLoader": "consul-config-loader-placeholder",
"consulConfigLoaderImage": "consul-config-loader-image-placeholder",
Expand Down Expand Up @@ -1784,9 +1775,6 @@ exports[`generator - app with microservice should match snapshot 1`] = `
"zipkin": "zipkin-placeholder",
"zipkinImage": "zipkin-image-placeholder",
"zipkinTag": "zipkin-tag-placeholder",
"zookeeper": "zookeeper-placeholder",
"zookeeperImage": "zookeeper-image-placeholder",
"zookeeperTag": "zookeeper-tag-placeholder",
},
"dockerServices": [
"app",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ Launch all your infrastructure by running: `docker compose up -d`.
<%_ } _%>
<%_ if (useKafka === true) { _%>
- Kafka
- Zookeeper
<%_ } _%>
<%_ if (usePulsar === true) { _%>
- Pulsar
Expand Down
16 changes: 7 additions & 9 deletions generators/docker-compose/templates/docker-compose.yml.ejs
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,17 @@ services:
<%- appsYaml[i] %>
<%_ } _%>
<%_ if (useKafka) { _%>
zookeeper:
image: <%- dockerContainers.zookeeper %>
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_SYNC_LIMIT: 2
kafka:
image: <%- dockerContainers.kafka %>
environment:
KAFKA_NODE_ID: 1
KAFKA_PROCESS_ROLES: broker,controller
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_BROKER_ID: 2
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://localhost:9093
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
#ports:
# - 9092:9092 # Uncomment to make Kafka available externally
<%_ } _%>
Expand Down
16 changes: 0 additions & 16 deletions generators/docker/__snapshots__/generator.spec.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,6 @@ services:
extends:
file: ./kafka.yml
service: kafka
zookeeper:
extends:
file: ./kafka.yml
service: zookeeper
",
"stateCleared": "modified",
},
Expand Down Expand Up @@ -365,10 +361,6 @@ services:
extends:
file: ./kafka.yml
service: kafka
zookeeper:
extends:
file: ./kafka.yml
service: zookeeper
",
"stateCleared": "modified",
},
Expand Down Expand Up @@ -743,10 +735,6 @@ services:
extends:
file: ./kafka.yml
service: kafka
zookeeper:
extends:
file: ./kafka.yml
service: zookeeper
",
"stateCleared": "modified",
},
Expand Down Expand Up @@ -856,10 +844,6 @@ services:
extends:
file: ./kafka.yml
service: kafka
zookeeper:
extends:
file: ./kafka.yml
service: zookeeper
",
"stateCleared": "modified",
},
Expand Down
5 changes: 1 addition & 4 deletions generators/docker/generator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,10 +228,7 @@ export default class DockerGenerator extends BaseApplicationGenerator {
);
}
if (application.dockerServices.includes('kafka')) {
source.addDockerExtendedServiceToApplicationAndServices(
{ serviceName: 'kafka' },
{ serviceFile: './kafka.yml', serviceName: 'zookeeper' },
);
source.addDockerExtendedServiceToApplicationAndServices({ serviceName: 'kafka' });
}
},

Expand Down
24 changes: 8 additions & 16 deletions generators/docker/templates/docker/kafka.yml.ejs
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,14 @@ services:
ports:
- 127.0.0.1:9092:9092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_ADVERTISED_HOST_NAME: kafka
<%_ if (backendTypeSpringBoot) { _%>
labels:
org.springframework.boot.ignore: true
<%_ } _%>
zookeeper:
image: <%- dockerContainers.zookeeper %>
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
KAFKA_NODE_ID: 1
KAFKA_PROCESS_ROLES: broker,controller
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://localhost:9093
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
<%_ if (backendTypeSpringBoot) { _%>
labels:
org.springframework.boot.ignore: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ dependencies:
"csvc-helm/templates/_helpers.tpl": {
"contents": "{{/* vim: set filetype=mustache: */}}
{{/*
Kafka and zookeeper customisation
Kafka customisation
*/}}
{{- define "kafka.name" -}}
{{- default "jhipster-kafka" -}}
Expand All @@ -126,17 +126,6 @@ Kafka and zookeeper customisation
{{- default "jhipster-kafka" -}}
{{- end -}}
{{- define "zookeeper.name" -}}
{{- default "jhipster-zookeeper" -}}
{{- end -}}
{{- define "zookeeper.fullname" -}}
{{- default "jhipster-zookeeper" -}}
{{- end -}}
{{- define "kafka.zookeeper.fullname" -}}
{{- default "jhipster-zookeeper" -}}
{{- end -}}
",
"stateCleared": "modified",
Expand All @@ -145,9 +134,6 @@ Kafka and zookeeper customisation
"contents": "kafka:
enabled: true
replicas: 3
zookeeper:
enabled: true
replicas: 3
",
"stateCleared": "modified",
},
Expand Down
13 changes: 1 addition & 12 deletions generators/kubernetes-helm/templates/csvc/helpers.tpl.ejs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{{/* vim: set filetype=mustache: */}}
<%_ if (useKafka) { _%>
{{/*
Kafka and zookeeper customisation
Kafka customisation
*/}}
{{- define "kafka.name" -}}
{{- default "jhipster-kafka" -}}
Expand All @@ -11,17 +11,6 @@ Kafka and zookeeper customisation
{{- default "jhipster-kafka" -}}
{{- end -}}

{{- define "zookeeper.name" -}}
{{- default "jhipster-zookeeper" -}}
{{- end -}}

{{- define "zookeeper.fullname" -}}
{{- default "jhipster-zookeeper" -}}
{{- end -}}

{{- define "kafka.zookeeper.fullname" -}}
{{- default "jhipster-zookeeper" -}}
{{- end -}}
<%_ } _%>
<%_ if (monitoringPrometheus) { _%>
{{/*
Expand Down
7 changes: 0 additions & 7 deletions generators/kubernetes-helm/templates/csvc/values.yml.ejs
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,6 @@ kafka:
podAnnotations:
sidecar.istio.io/inject: "false"
<%_ } _%>
zookeeper:
enabled: true
replicas: 3
<%_ if (istio) { _%>
podAnnotations:
sidecar.istio.io/inject: "false"
<%_ } _%>
<%_ } _%>
<%_ if (monitoringPrometheus) { _%>
prometheus:
Expand Down
68 changes: 16 additions & 52 deletions generators/kubernetes/__snapshots__/kubernetes.spec.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -203,14 +203,24 @@ spec:
- name: kafka
image: kafka-placeholder
env:
- name: KAFKA_NODE_ID
value: 1
- name: KAFKA_PROCESS_ROLES
value: broker,controller
- name: KAFKA_ADVERTISED_LISTENERS
value: 'PLAINTEXT://jhipster-kafka.default.svc.cluster.local:9092'
- name: KAFKA_ZOOKEEPER_CONNECT
value: 'jhipster-zookeeper.default.svc.cluster.local:2181'
- name: KAFKA_BROKER_ID
value: '2'
- name: KAFKA_LISTENERS
value: PLAINTEXT://:9092,CONTROLLER://localhost:9093
- name: KAFKA_CONTROLLER_LISTENER_NAMES
value: CONTROLLER
- name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
- name: KAFKA_CONTROLLER_QUORUM_VOTERS
value: 1@localhost:9093
- name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS
value: 0
- name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
value: '1'
value: 1
ports:
- containerPort: 9092
resources:
Expand All @@ -231,53 +241,7 @@ spec:
app: jhipster-kafka
ports:
- port: 9092
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: jhipster-zookeeper
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: jhipster-zookeeper
template:
metadata:
labels:
app: jhipster-zookeeper
spec:
containers:
- name: zookeeper
image: zookeeper-placeholder
env:
- name: ZOOKEEPER_CLIENT_PORT
value: '2181'
- name: ZOOKEEPER_TICK_TIME
value: '2000'
- name: ZOOKEEPER_SYNC_LIMIT
value: '2'
ports:
- containerPort: 2181
resources:
requests:
memory: "512Mi"
cpu: "500m"
limits:
memory: "1Gi"
cpu: "1"
---
apiVersion: v1
kind: Service
metadata:
name: jhipster-zookeeper
namespace: default
spec:
selector:
app: jhipster-zookeeper
ports:
- port: 2181
",
",
"stateCleared": "modified",
},
"samplekafka-k8s/samplekafka-deployment.yml": {
Expand Down
Loading

0 comments on commit 03d45f9

Please sign in to comment.