Run kafka-connect docker image within scaffold

Author: sausage-todd

scripts/scaffold.yaml

@@ -140,7 +140,7 @@ services:
       - crowd-bridge
 
   kafka:
-    image: bitnami/kafka:latest
+    image: bitnami/kafka:3.8.0-debian-12-r6
     restart: unless-stopped
     environment:
       - KAFKA_CFG_NODE_ID=0
@@ -161,6 +161,23 @@ services:
     networks:
       - crowd-bridge
 
+  kafka-connect:
+    build:
+      context: scaffold/kafka-connect
+    restart: unless-stopped
+    entrypoint:
+      - connect-standalone
+      - /etc/kafka-connect/worker-local.properties
+      - /etc/kafka-connect/console-local-sink.properties
+      - /etc/kafka-connect/questdb-local-sink.properties
+    volumes:
+      - kafka-connect-dev:/storage
+      - ./scaffold/kafka-connect/worker-local.properties:/etc/kafka-connect/worker-local.properties
+      - ./scaffold/kafka-connect/console-local-sink.properties:/etc/kafka-connect/console-local-sink.properties
+      - ./scaffold/kafka-connect/questdb-local-sink.properties:/etc/kafka-connect/questdb-local-sink.properties
+    networks:
+      - crowd-bridge
+
   temporal:
     build:
       context: scaffold/temporal
@@ -182,3 +199,4 @@ volumes:
   opensearch-dev:
   s3-dev:
   redis-dev:
+  kafka-connect-dev:

scripts/scaffold/kafka-connect/console-local-sink.properties

@@ -0,0 +1,6 @@
+name=console-sink
+connector.class=FileStreamSinkConnector
+tasks.max=1
+topics=activities
+value.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter.schemas.enable=false

scripts/scaffold/kafka-connect/questdb-local-sink.properties

@@ -0,0 +1,12 @@
+name=questdb-sink
+client.conf.string=http::addr=questdb:9000;
+topics=activities
+table=activities
+connector.class=io.questdb.kafka.QuestDBSinkConnector
+value.converter=org.apache.kafka.connect.json.JsonConverter
+include.key=false
+key.converter=org.apache.kafka.connect.storage.StringConverter
+timestamp.field.name=timestamp
+timestamp.string.fields=createdAt,updatedAt
+timestamp.string.format=yyyy-MM-ddTHH:mm:ss.SSSZ
+value.converter.schemas.enable=false

scripts/scaffold/kafka-connect/worker-local.properties

@@ -0,0 +1,13 @@
+bootstrap.servers=kafka:9092
+group.id=kafka-connect
+
+key.converter=org.apache.kafka.connect.storage.StringConverter
+value.converter=org.apache.kafka.connect.json.JsonConverter
+internal.key.converter=org.apache.kafka.connect.storage.StringConverter
+internal.value.converter=org.apache.kafka.connect.json.JsonConverter
+key.converter.schemas.enable=true
+value.converter.schemas.enable=true
+
+offset.storage.file.filename=/storage/connect.offsets
+offset.flush.interval.ms=10000
+plugin.path=/usr/share/java,/usr/share/filestream-connectors,/usr/share/confluent-hub-components