Debezium Connect with schema registry - avro

Question

With my Connector for debezium
{
"name": "arctype-connector-hint",
"config": {
"connector.class": "io.debezium.connector.postgresql.PostgresConnector",
"tasks.max": "1",
"plugin.name": "pgoutput",
"database.hostname": "db",
"database.port": "5432",
"database.user": "postgres",
"database.password": "arctype",
"database.dbname": "test",
"database.server.name": "ARCTYPE",
"table.include.list": "public.outbox_wal",
"transforms": "outbox,setValueSchema,selective",
"transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter",
"transforms.outbox.route.topic.replacement": "${routedByValue}",
"transforms.outbox.table.fields.additional.placement": "type:header:eventType,id:header:messageId",
"transforms.outbox.table.expand.json.payload": "true",
"transforms.selective.predicate": "IsOutboxTable",
"transforms.selective.type": "io.debezium.transforms.outbox.EventRouter",
"predicates": "IsOutboxTable",
"predicates.IsOutboxTable.type": "org.apache.kafka.connect.transforms.predicates.TopicNameMatches",
"predicates.IsOutboxTable.pattern": "asset",
"publication.autocreate.mode": "filtered",
"topic.creation.default.replication.factor": 1,
"topic.creation.default.partitions": 1,
"topic.creation.default.compression.type": "lz4",
"transforms.setValueSchema.type": "org.apache.kafka.connect.transforms.SetSchemaMetadata$Value",
"transforms.setValueSchema.schema.name": "io.confluent.connect.avro.ConnectDefault",
"heartbeat.topics.prefix": "debezium-heartbeat",
"key.converter.schemas.enable": "true",
"key.converter": "io.confluent.connect.avro.AvroConverter",
"key.converter.schema.registry.url": "http://schema-registry:8081",
"value.converter": "io.confluent.connect.avro.AvroConverter",
"value.converter.schema.registry.url": "http://schema-registry:8081",
"internal.key.converter": "org.apache.kafka.connect.json.JsonConverter",
"internal.value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.connect.meta.data": "false",
"connect.meta.data": "false",
"heartbeat.interval.ms": "5000",
"heartbeat.action.query": "INSERT INTO kafka_connect_heartbeat (id, last_heartbeat_time) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET last_heartbeat_time=EXCLUDED.last_heartbeat_time;",
"heartbeat.writeback.enabled": true,
"heartbeat.writeback.table": "public.kafka_connect_heartbeat",
"slot.name": "testpgoutput"
}
}
I am getting error
org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler
connect_1 | at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:223)
connect_1 | at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:149)
connect_1 | at org.apache.kafka.connect.runtime.WorkerSourceTask.convertTransformedRecord(WorkerSourceTask.java:330)
connect_1 | at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:356)
connect_1 | at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:258)
connect_1 | at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:188)
connect_1 | at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:243)
connect_1 | at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
connect_1 | at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
connect_1 | at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
connect_1 | at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
connect_1 | at java.base/java.lang.Thread.run(Thread.java:829)
connect_1 | Caused by: org.apache.kafka.connect.errors.DataException: Failed to serialize Avro data from topic asset :
connect_1 | at io.confluent.connect.avro.AvroConverter.fromConnectData(AvroConverter.java:93)
connect_1 | at org.apache.kafka.connect.storage.Converter.fromConnectData(Converter.java:64)
connect_1 | at org.apache.kafka.connect.runtime.WorkerSourceTask.lambda$convertTransformedRecord$3(WorkerSourceTask.java:330)
connect_1 | at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:173)
connect_1 | at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:207)
connect_1 | ... 11 more
connect_1 | Caused by: org.apache.kafka.common.errors.SerializationException: Error serializing Avro message
connect_1 | at io.confluent.kafka.serializers.AbstractKafkaAvroSerializer.serializeImpl(AbstractKafkaAvroSerializer.java:154)
connect_1 | at io.confluent.connect.avro.AvroConverter$Serializer.serialize(AvroConverter.java:153)
connect_1 | at io.confluent.connect.avro.AvroConverter.fromConnectData(AvroConverter.java:86)
connect_1 | ... 15 more
connect_1 | Caused by: org.apache.avro.SchemaParseException: Can't redefine: io.confluent.connect.avro.ConnectDefault
connect_1 | at org.apache.avro.Schema$Names.put(Schema.java:1542)
connect_1 | at org.apache.avro.Schema$NamedSchema.writeNameRef(Schema.java:805)
connect_1 | at org.apache.avro.Schema$RecordSchema.toJson(Schema.java:967)
connect_1 | at org.apache.avro.Schema$UnionSchema.toJson(Schema.java:1234)
connect_1 | at org.apache.avro.Schema$RecordSchema.fieldsToJson(Schema.java:995)
connect_1 | at org.apache.avro.Schema$RecordSchema.toJson(Schema.java:979)
connect_1 | at org.apache.avro.Schema$UnionSchema.toJson(Schema.java:1234)
connect_1 | at org.apache.avro.Schema.toString(Schema.java:419)
connect_1 | at org.apache.avro.Schema.toString(Schema.java:410)
connect_1 | at io.confluent.kafka.schemaregistry.avro.AvroSchema.canonicalString(AvroSchema.java:151)
connect_1 | at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.registerAndGetId(CachedSchemaRegistryClient.java:257)
connect_1 | at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.register(CachedSchemaRegistryClient.java:366)
connect_1 | at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.register(CachedSchemaRegistryClient.java:337)
connect_1 | at io.confluent.kafka.serializers.AbstractKafkaAvroSerializer.serializeImpl(AbstractKafkaAvroSerializer.java:115)
connect_1 | ... 17 more
1.) First I Created Connector with First Json with confluent schema registry.
2.) When message has been produced from outbox_wal table to kafka but schema registry is giving exception
If anybody faced this please share
3.) Exception is also pasted above.
Message type we are using is AVRO