Elasticsearch Version
9.1.3
Installed Plugins
No response
Java Version
bundled
OS Version
ECH
Problem Description
Watchdog output (after 5 seconds of no progress):
Hot threads at 2026-02-27T14:51:55.574Z, interval=500ms, busiestThreads=500, ignoreIdleThreads=false:
100.2% [cpu=100.2%, idle=0.0%] (501ms out of 500ms) cpu usage by thread 'elasticsearch[instance-0000000065][transport_worker][T#1]'
2/10 snapshots sharing following 36 elements
app/org.elasticsearch.xcontent@9.1.3/org.elasticsearch.xcontent.XContentBuilder.map(XContentBuilder.java:1062)
app/org.elasticsearch.xcontent@9.1.3/org.elasticsearch.xcontent.XContentBuilder.field(XContentBuilder.java:1058)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.action.admin.indices.get.GetIndexResponse.lambda$toXContentChunked$4(GetIndexResponse.java:209)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.action.admin.indices.get.GetIndexResponse$$Lambda/0x000000001e2e54b8.toXContent(Unknown Source)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.rest.ChunkedRestResponseBodyPart$1.encodeChunk(ChunkedRestResponseBodyPart.java:161)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.rest.RestController$EncodedLengthTrackingChunkedRestResponseBodyPart.encodeChunk(RestController.java:1026)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.writeChunk(Netty4HttpPipeliningHandler.java:436)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.doFlush(Netty4HttpPipeliningHandler.java:414)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelWritabilityChanged(Netty4HttpPipeliningHandler.java:371)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:526)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:505)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.fireChannelWritabilityChanged(AbstractChannelHandlerContext.java:498)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.transport.netty4.Netty4WriteThrottlingHandler.channelWritabilityChanged(Netty4WriteThrottlingHandler.java:138)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:526)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:505)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.fireChannelWritabilityChanged(AbstractChannelHandlerContext.java:498)
io.netty.transport@4.1.118.Final/io.netty.channel.DefaultChannelPipeline$HeadContext.channelWritabilityChanged(DefaultChannelPipeline.java:1380)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:524)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:505)
io.netty.transport@4.1.118.Final/io.netty.channel.DefaultChannelPipeline.fireChannelWritabilityChanged(DefaultChannelPipeline.java:880)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.fireChannelWritabilityChanged(ChannelOutboundBuffer.java:659)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.setWritable(ChannelOutboundBuffer.java:625)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.decrementPendingOutboundBytes(ChannelOutboundBuffer.java:207)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.remove(ChannelOutboundBuffer.java:304)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.removeBytes(ChannelOutboundBuffer.java:383)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.transport.netty4.CopyBytesSocketChannel.doWrite(CopyBytesSocketChannel.java:103)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:929)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.forceFlush(AbstractNioChannel.java:366)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:790)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:697)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:660)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
io.netty.common@4.1.118.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
io.netty.common@4.1.118.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@24.0.2/java.lang.Thread.runWith(Thread.java:1460)
java.base@24.0.2/java.lang.Thread.run(Thread.java:1447)
8/10 snapshots sharing following 37 elements
app/org.elasticsearch.server@9.1.3/org.elasticsearch.common.xcontent.XContentHelper.parseToType(XContentHelper.java:212)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:146)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.cluster.metadata.MappingMetadata.sourceAsMap(MappingMetadata.java:129)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.action.admin.indices.get.GetIndexResponse.lambda$toXContentChunked$4(GetIndexResponse.java:209)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.action.admin.indices.get.GetIndexResponse$$Lambda/0x000000001e2e54b8.toXContent(Unknown Source)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.rest.ChunkedRestResponseBodyPart$1.encodeChunk(ChunkedRestResponseBodyPart.java:161)
app/org.elasticsearch.server@9.1.3/org.elasticsearch.rest.RestController$EncodedLengthTrackingChunkedRestResponseBodyPart.encodeChunk(RestController.java:1026)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.writeChunk(Netty4HttpPipeliningHandler.java:436)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.doFlush(Netty4HttpPipeliningHandler.java:414)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelWritabilityChanged(Netty4HttpPipeliningHandler.java:371)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:526)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:505)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.fireChannelWritabilityChanged(AbstractChannelHandlerContext.java:498)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.transport.netty4.Netty4WriteThrottlingHandler.channelWritabilityChanged(Netty4WriteThrottlingHandler.java:138)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:526)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:505)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.fireChannelWritabilityChanged(AbstractChannelHandlerContext.java:498)
io.netty.transport@4.1.118.Final/io.netty.channel.DefaultChannelPipeline$HeadContext.channelWritabilityChanged(DefaultChannelPipeline.java:1380)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:524)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:505)
io.netty.transport@4.1.118.Final/io.netty.channel.DefaultChannelPipeline.fireChannelWritabilityChanged(DefaultChannelPipeline.java:880)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.fireChannelWritabilityChanged(ChannelOutboundBuffer.java:659)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.setWritable(ChannelOutboundBuffer.java:625)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.decrementPendingOutboundBytes(ChannelOutboundBuffer.java:207)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.remove(ChannelOutboundBuffer.java:304)
io.netty.transport@4.1.118.Final/io.netty.channel.ChannelOutboundBuffer.removeBytes(ChannelOutboundBuffer.java:383)
org.elasticsearch.transport.netty4@9.1.3/org.elasticsearch.transport.netty4.CopyBytesSocketChannel.doWrite(CopyBytesSocketChannel.java:103)
io.netty.transport@4.1.118.Final/io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:929)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.forceFlush(AbstractNioChannel.java:366)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:790)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:697)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:660)
io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
io.netty.common@4.1.118.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
io.netty.common@4.1.118.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@24.0.2/java.lang.Thread.runWith(Thread.java:1460)
java.base@24.0.2/java.lang.Thread.run(Thread.java:1447)
Steps to Reproduce
Search logs at Feb 27, 2026 @ 14:51:56.236
Elasticsearch Version
9.1.3
Installed Plugins
No response
Java Version
bundled
OS Version
ECH
Problem Description
Watchdog output (after 5 seconds of no progress):
Steps to Reproduce
Search logs at Feb 27, 2026 @ 14:51:56.236