How to print the exception in custom log messages when throwing Retryable errors to DefaultErrorHandler DLT? - spring-kafka

Question

Ive implemented a KafkaConsumerFactory in which for the errors (Retryable error) I throw the exception. I want these exceptions to be printed an a proper logging format like sflf4j(LOGGER.ERROR) and not print on the console.
The ask is I want to print the errors in a proper log format instead of throwing in console with a huge stacktrace.
Below is the code snippet for KakaConsumerConfigFactory.java that has DLT Implementation to process and push error records to DLT topic.
package org.abc;
import org.apache.kafka.common.TopicPartition;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.kafka.ConcurrentKafkaListenerContainerFactoryConfigurer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
import org.springframework.kafka.listener.DefaultErrorHandler;
import org.springframework.util.backoff.FixedBackOff;
import com.azure.cosmos.CosmosException;
import com.azure.spring.data.cosmos.exception.CosmosAccessException;
import com.fasterxml.jackson.core.JsonProcessingException;
#Configuration
public class KafkaListenerConfig {
#Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<Object, Object> kafkaListenerContainerFactory(
ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
ConsumerFactory<Object, Object> kafkaConsumerFactory, KafkaTemplate<Object, Object> template) {
ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
configurer.configure(factory, kafkaConsumerFactory);
var recoverer = new DeadLetterPublishingRecoverer(template,
(record, ex) -> new TopicPartition("abc_dlt", record.partition()));
var errorHandler = getDLTDetails(recoverer);
factory.setCommonErrorHandler(errorHandler);
return factory;
}
#Bean(name = "kafkaListenerContainerFactoryForDLT")
public ConcurrentKafkaListenerContainerFactory<Object, Object> kafkaListenerContainerFactoryForDLT(
ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
ConsumerFactory<Object, Object> kafkaConsumerFactory, KafkaTemplate<Object, Object> template) {
ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
configurer.configure(factory, kafkaConsumerFactory);
var errorHandler = new DefaultErrorHandler(new FixedBackOff(3, 2));
factory.setCommonErrorHandler(errorHandler);
factory.getContainerProperties().setIdleEventInterval(70000L);
return factory;
}
private DefaultErrorHandler getDLTDetails(DeadLetterPublishingRecoverer recoverer) {
var errorHandler = new DefaultErrorHandler(recoverer, new FixedBackOff(3, 2));
errorHandler.addNotRetryableExceptions(JsonProcessingException.class);
errorHandler.addRetryableExceptions(DBException.class, DataAccessException.class);//Sample db exceptions
errorHandler.setCommitRecovered(true);
return errorHandler;
}
}
Below is the class in which the error is throw from listener.java
#Component
public class Listener {
#Autowired
private LogUtil log; //Custom logger
public Listener (final LogUtil log) {
this.log= log;
}
#KafkaListener(id = "hub_topic", topics = "hub_topic", groupId = "hub_topic_0", containerFactory = "kafkaListenerContainerFactory", clientIdPrefix = "hub_topic")
public void listen(ConsumerRecord<String, String> consumerRecord, Acknowledgment ack) {
log.printLog("INFO",consumerRecord.value());
saveToDB(consumerRecord.value()); //Method to persist obtained request to Database Columns //: id, String Payload-Consumer record
ack.acknowledge();
}
}

Answer 1

You can prevent the stack trace logs by setting the error handler's logLevel property to TRACE.
Then, subclass the error hander and do whatever logging you want do yourself after calling the super class methods.
EDIT
For example:
#Bean
CommonErrorHandler eh() {
DefaultErrorHandler eh = new DefaultErrorHandler((rec, ex) -> {
System.out.println("Recovered " + KafkaUtils.format(rec));
}, new FixedBackOff(3000L, 2)) {
#Override
public void handleRemaining(Exception thrownException, List<ConsumerRecord<?, ?>> records,
Consumer<?, ?> consumer, MessageListenerContainer container) {
try {
super.handleRemaining(thrownException, records, consumer, container);
// log record was recovered
}
catch (RuntimeException ex) {
// log retrying or recovery failed
throw ex;
}
}
};
eh.setLogLevel(Level.TRACE);
return eh;
}