Checkpointing Performance Issue on azure event hubs using kafka.js - azure-eventhub

Question

We are using kafka.js event client with event hubs and we are seeing performance issues with kafka.js. The event client was originally running on Azure SDK event client for event hubs and we recently upgraded to kafka.js.
The issue is it's taking longer time to checkpoint before committing the event.
Azure SDK used to take 15ms to checkpoint (using external Azure storage account outside of event hubs) vs new kafka.js event client relying on Azure event hubs internal storage for checkpointing but taking more than 1 second to checkpoint. Extremely slow.
We are using eachMessage and checkpointing after each message to ensure data consistency but Azure event hubs throttles 4 calls/second per partition and each checkpoint call takes close to 1 second due to shared internal checkpoint storage used by event hubs.
We tried using eachBatch with autoCommit but it seems to be committing entire batch of messages in case if it fails during batch processing.
Anyone noticed similar issue elsewhere? appreciate any feedback or suggestion to solve this.
Here is the code snippet:
/**
* #method consume()
*/
public async consume() {
await this.client.run({
eachBatchAutoResolve: this.configuration.eachBatchAutoResolve, // default to false so consumer does not auto-resolve the offsets
autoCommitThreshold: this.configuration.autoCommitThreshold,
partitionsConsumedConcurrently: this.configuration.partitionsConsumedConcurrently,
eachBatch: async ({ batch, resolveOffset, heartbeat }) => {
if (this.shutdownContext != null) {
// prevent processing new events during a shutdown.
return;
}
const { topic, partition } = batch;
const partitionId = `${topic}-${partition}`;
if (this.partitionTracker.isPartitionProcessing(partitionId)) {
// prevent concurrent event processing on the same partition.
return;
}
this.partitionTracker.startProcessing(partitionId);
this.genericLogger.debug(
`batch last offset: ${batch.lastOffset()} partitionId: ${partitionId} highWatermark: ${
batch.highWatermark
} offsetLag: ${batch.offsetLag()}`,
);
/* eslint-disable no-await-in-loop, no-restricted-syntax */
for (const message of batch.messages) {
let event: Event<any>;
const partitionKey = message.key ? message.key.toString() : message.key;
const { offset, timestamp } = message;
// build event context
const context: EventContext = {
topic,
partition,
partitionKey,
offset,
timestamp,
};
try {
// transform kafka message to event
event = await this.transform(message);
} catch (error) {
const id = uuid.v4();
this.genericLogger.crit(
`Event failed consumer transformation. Please check the Dead Letter topic for a version of the event:${id}`,
);
const poisonousEvent: Event<any> = {
id,
correlationId: { id, origin: uuid.v4() },
data: message.value,
type: 'ca.event.poisonousmessage.created.v1',
};
await this.dlqProducer.send(
poisonousEvent,
this.configuration.dlqTopic,
this.genericLogger,
);
try {
resolveOffset(message.offset);
await heartbeat();
// await commitOffsetsIfNecessary();
this.partitionTracker.stopProcessing(partitionId);
await this.shutdownCheck(context);
} catch (commitError) {
await this.signalShutdown(
ProcessExceptionType.FAILURE_TO_PROCESS_AND_COMMIT,
commitError,
);
await this.shutdown({
reason: ProcessExceptionType.FAILURE_TO_PROCESS_AND_COMMIT,
error: commitError,
eventContext: context,
});
}
break;
}
const contextLogger = getLoggerFactory(
this.configuration.loggerOptions,
'event-client-consumer',
)(getEventMetaData(event));
const eventMessageContext = `[eventHubName:${topic} partition:${partition} offset:${offset} partitionKey:${partitionKey} timestampUTC:${timestamp}]`;
contextLogger.debug(`received new message on ${eventMessageContext}.`);
try {
// heartbeat
await heartbeat();
} catch (error) {
error.message = `error while manually sending a heartbeat: ${error.message}`;
if (error.message.includes(REBALANCING)) {
contextLogger.warn(error.message);
} else {
contextLogger.error(error.message);
}
break;
}
try {
// process
await this.process(event, context, contextLogger, heartbeat);
resolveOffset(message.offset);
// commit
// await this.commit(
// event,
// context,
// contextLogger,
// message.offset,
// resolveOffset,
// commitOffsetsIfNecessary,
// );
} catch (error) {
if (error.message.includes(REBALANCING)) {
contextLogger.warn(
`rebalancing error during processing or committing: ${error.message}`,
);
throw error;
}
const exception = error as Exception;
switch (exception.type) {
case ProcessExceptionType.COMMIT: {
contextLogger.error(
`[Event Processing Error] failed to commit processed event on ${eventMessageContext}. error: ${error.message}.`,
error,
);
// failed to commit message after retry. sleeping.
await this.sleep(context, error as Error, exception.type, heartbeat);
// await this.signalShutdown(exception.type, error);
// await this.shutdown({ reason: exception.type, error, eventContext: context });
throw error;
}
case ProcessExceptionType.SHUTDOWN_NO_COMMIT: {
// shutdown with no commit. most likely due to non retryable circuit breaker exception.
await this.sleep(context, error as Error, exception.type, heartbeat);
// await this.signalShutdown(exception.type, error);
// await this.shutdown({ reason: exception.type, error, eventContext: context });
throw error;
}
case ProcessExceptionType.SLEEP_NO_COMMIT: {
// return early for sleep cycle.
throw error;
}
default: {
if (this.configuration.continueOnFailedEventProcessing) {
contextLogger.crit(
`[Event Processing Error] Error in user defined code when processing event on ${eventMessageContext} beyond process loop retry limit error: ${error.message}. Commiting event to continue processing new events.`,
error,
);
try {
resolveOffset(message.offset);
await heartbeat();
// await commitOffsetsIfNecessary();
this.partitionTracker.stopProcessing(partitionId);
await this.shutdownCheck(context);
} catch (commitError) {
await this.sleep(context, commitError as Error, exception.type, heartbeat);
// await this.signalShutdown(
// ProcessExceptionType.FAILURE_TO_PROCESS_AND_COMMIT,
// commitError,
// );
// await this.shutdown({
// reason: ProcessExceptionType.FAILURE_TO_PROCESS_AND_COMMIT,
// error: commitError,
// eventContext: context,
// });
}
} else {
contextLogger.crit(
`[Event Processing Error] Error in user defined code when processing event on ${eventMessageContext} beyond process loop retry limit error: ${error.message}. Initiating sleep.`,
error,
);
await this.sleep(context, error as Error, exception.type, heartbeat);
// await this.signalShutdown(
// ProcessExceptionType.FAILURE_TO_PROCESS_AND_COMMIT,
// error,
// );
// await this.shutdown({
// reason: ProcessExceptionType.FAILURE_TO_PROCESS_AND_COMMIT,
// error,
// eventContext: context,
// });
}
}
}
throw error;
}
}
this.genericLogger.debug(
`batch last offset: ${batch.lastOffset()} partitionId: ${partitionId} highWatermark: ${
batch.highWatermark
} offsetLag: ${batch.offsetLag()}`,
);
this.genericLogger.debug(
`resolved batch of ${batch.messages.length} messages partitionId: ${partitionId}`,
);
this.partitionTracker.stopProcessing(partitionId);
await this.shutdownCheck();
},
});
}