2017-02-04T09:27:55,872][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1936] overhead, spent [622ms] collecting in the last [1s] [2017-02-04T09:28:03,183][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1937][56] duration [6.4s], collections [1]/[7.3s], total [6.4s]/[1.3m], memory [6.1gb]->[6gb]/[6.9gb], all_pools {[young] [7.7mb]->[9.4mb]/[266.2mb]}{[survivor] [33.2mb]->[0b]/[33.2mb]}{[old] [6gb]->[6gb]/[6.6gb]} [2017-02-04T09:28:03,183][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1937] overhead, spent [6.8s] collecting in the last [7.3s] [2017-02-04T09:28:10,233][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1938][57] duration [6.4s], collections [1]/[7s], total [6.4s]/[1.4m], memory [6gb]->[6.2gb]/[6.9gb], all_pools {[young] [9.4mb]->[8.1mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6gb]->[6.2gb]/[6.6gb]} [2017-02-04T09:28:10,233][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1938] overhead, spent [6.5s] collecting in the last [7s] [2017-02-04T09:28:17,104][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1939][58] duration [6.6s], collections [1]/[6.8s], total [6.6s]/[1.5m], memory [6.2gb]->[6.2gb]/[6.9gb], all_pools {[young] [8.1mb]->[16mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.2gb]->[6.2gb]/[6.6gb]} [2017-02-04T09:28:17,104][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1939] overhead, spent [6.6s] collecting in the last [6.8s] [2017-02-04T09:28:23,941][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1940][59] duration [6.6s], collections [1]/[6.8s], total [6.6s]/[1.6m], memory [6.2gb]->[6.5gb]/[6.9gb], all_pools {[young] [16mb]->[40.4mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.2gb]->[6.5gb]/[6.6gb]} [2017-02-04T09:28:23,942][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1940] overhead, spent [6.6s] collecting in the last [6.8s] [2017-02-04T09:28:28,949][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1941] overhead, spent [4.9s] collecting in the last [5s] [2017-02-04T09:28:35,694][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1942][61] duration [6.6s], collections [1]/[6.7s], total [6.6s]/[1.8m], memory [6.5gb]->[6.3gb]/[6.9gb], all_pools {[young] [48.7mb]->[16.3mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.5gb]->[6.3gb]/[6.6gb]} [2017-02-04T09:28:35,694][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1942] overhead, spent [6.6s] collecting in the last [6.7s] [2017-02-04T09:28:42,608][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1943][62] duration [6.7s], collections [1]/[6.9s], total [6.7s]/[1.9m], memory [6.3gb]->[6.6gb]/[6.9gb], all_pools {[young] [16.3mb]->[62.1mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.3gb]->[6.5gb]/[6.6gb]} [2017-02-04T09:28:42,608][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1943] overhead, spent [6.7s] collecting in the last [6.9s] [2017-02-04T09:28:47,461][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1944] overhead, spent [4.7s] collecting in the last [4.8s] [2017-02-04T09:28:53,905][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1945][64] duration [6.2s], collections [1]/[6.4s], total [6.2s]/[2.1m], memory [6.2gb]->[6.4gb]/[6.9gb], all_pools {[young] [2.3mb]->[44mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.2gb]->[6.4gb]/[6.6gb]} [2017-02-04T09:28:53,905][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1945] overhead, spent [6.2s] collecting in the last [6.4s] [2017-02-04T09:28:57,914][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1946] overhead, spent [3.9s] collecting in the last [4s] [2017-02-04T09:29:03,615][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1947][66] duration [5.6s], collections [1]/[5.7s], total [5.6s]/[2.2m], memory [6.4gb]->[6.5gb]/[6.9gb], all_pools {[young] [2.6mb]->[83.8mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.4gb]->[6.4gb]/[6.6gb]} [2017-02-04T09:29:03,615][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1947] overhead, spent [5.6s] collecting in the last [5.7s] [2017-02-04T09:29:07,970][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1948] overhead, spent [4.2s] collecting in the last [4.3s] [2017-02-04T09:29:13,779][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1949][68] duration [5.5s], collections [1]/[5.8s], total [5.5s]/[2.4m], memory [6.2gb]->[6.4gb]/[6.9gb], all_pools {[young] [16mb]->[16mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.2gb]->[6.4gb]/[6.6gb]} [2017-02-04T09:29:13,779][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1949] overhead, spent [5.5s] collecting in the last [5.8s] [2017-02-04T09:29:18,560][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1950] overhead, spent [4.6s] collecting in the last [4.7s] [2017-02-04T09:29:25,211][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1951][70] duration [6.4s], collections [1]/[6.6s], total [6.4s]/[2.6m], memory [6.4gb]->[6.6gb]/[6.9gb], all_pools {[young] [16mb]->[8.7mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.4gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:29:25,212][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1951] overhead, spent [6.5s] collecting in the last [6.6s] [2017-02-04T09:29:29,235][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1952] overhead, spent [3.9s] collecting in the last [4s] [2017-02-04T09:29:35,020][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1953][72] duration [5.6s], collections [1]/[5.7s], total [5.6s]/[2.7m], memory [6.7gb]->[6.5gb]/[6.9gb], all_pools {[young] [97.3mb]->[45.6mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.6gb]->[6.5gb]/[6.6gb]} [2017-02-04T09:29:35,021][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1953] overhead, spent [5.6s] collecting in the last [5.7s] [2017-02-04T09:29:39,928][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1954] overhead, spent [4.8s] collecting in the last [4.9s] [2017-02-04T09:29:46,764][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1955][74] duration [6.7s], collections [1]/[6.8s], total [6.7s]/[2.9m], memory [6.7gb]->[6.7gb]/[6.9gb], all_pools {[young] [134.8mb]->[129mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:29:46,764][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1955] overhead, spent [6.7s] collecting in the last [6.8s] [2017-02-04T09:29:51,157][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1956] overhead, spent [4.3s] collecting in the last [4.3s] [2017-02-04T09:29:56,866][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1957][76] duration [5.5s], collections [1]/[5.7s], total [5.5s]/[3.1m], memory [6.7gb]->[6.7gb]/[6.9gb], all_pools {[young] [67.3mb]->[120.1mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:29:56,866][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1957] overhead, spent [5.5s] collecting in the last [5.7s] [2017-02-04T09:30:01,910][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1958] overhead, spent [4.9s] collecting in the last [5s] [2017-02-04T09:30:07,654][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1959][78] duration [5.6s], collections [1]/[5.7s], total [5.6s]/[3.3m], memory [6.7gb]->[6.8gb]/[6.9gb], all_pools {[young] [116mb]->[204.6mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:30:07,654][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1959] overhead, spent [5.6s] collecting in the last [5.7s] [2017-02-04T09:30:11,602][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1960] overhead, spent [3.9s] collecting in the last [3.9s] [2017-02-04T09:30:18,221][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1961][80] duration [6.5s], collections [1]/[6.6s], total [6.5s]/[3.5m], memory [6.8gb]->[6.9gb]/[6.9gb], all_pools {[young] [182.7mb]->[243.2mb]/[266.2mb]}{[survivor] [0b]->[0b]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:30:18,221][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1961] overhead, spent [6.5s] collecting in the last [6.6s] [2017-02-04T09:30:23,227][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1962] overhead, spent [4.9s] collecting in the last [5s] [2017-02-04T09:30:30,150][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1963][82] duration [6.9s], collections [1]/[6.9s], total [6.9s]/[3.7m], memory [6.9gb]->[6.9gb]/[6.9gb], all_pools {[young] [264mb]->[240.1mb]/[266.2mb]}{[survivor] [32.9mb]->[0b]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:30:30,150][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1963] overhead, spent [6.9s] collecting in the last [6.9s] [2017-02-04T09:30:39,968][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1964] overhead, spent [9.7s] collecting in the last [9.8s] [2017-02-04T09:30:46,850][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1965][85] duration [6.8s], collections [1]/[6.8s], total [6.8s]/[3.9m], memory [6.9gb]->[6.9gb]/[6.9gb], all_pools {[young] [257.2mb]->[263.8mb]/[266.2mb]}{[survivor] [16mb]->[16mb]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:30:46,850][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1965] overhead, spent [6.8s] collecting in the last [6.8s] [2017-02-04T09:30:51,795][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1966] overhead, spent [4.9s] collecting in the last [4.9s] [2017-02-04T09:31:10,855][ERROR][o.e.t.n.Netty4Utils ] fatal error on the network layer at org.elasticsearch.transport.netty4.Netty4Utils.maybeDie(Netty4Utils.java:141) at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.exceptionCaught(Netty4HttpRequestHandler.java:76) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelHandlerAdapter.exceptionCaught(ChannelHandlerAdapter.java:78) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.DefaultChannelPipeline$HeadContext.exceptionCaught(DefaultChannelPipeline.java:1301) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.DefaultChannelPipeline.fireExceptionCaught(DefaultChannelPipeline.java:914) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.handleReadException(AbstractNioByteChannel.java:99) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:140) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:651) at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:536) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:490) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:450) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at java.lang.Thread.run(Thread.java:745) [2017-02-04T09:31:45,739][INFO ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][old][1967][88] duration [10.4s], collections [2]/[10.4s], total [10.4s]/[4.2m], memory [6.9gb]->[6.9gb]/[6.9gb], all_pools {[young] [254.3mb]->[265.6mb]/[266.2mb]}{[survivor] [16mb]->[32mb]/[33.2mb]}{[old] [6.6gb]->[6.6gb]/[6.6gb]} [2017-02-04T09:31:45,740][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1967] overhead, spent [10.4s] collecting in the last [10.4s] [2017-02-04T09:31:45,740][ERROR][o.e.t.n.Netty4Utils ] fatal error on the network layer at org.elasticsearch.transport.netty4.Netty4Utils.maybeDie(Netty4Utils.java:141) at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.exceptionCaught(Netty4MessageChannelHandler.java:83) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.DefaultChannelPipeline$HeadContext.exceptionCaught(DefaultChannelPipeline.java:1301) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.DefaultChannelPipeline.fireExceptionCaught(DefaultChannelPipeline.java:914) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.handleReadException(AbstractNioByteChannel.java:99) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:140) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:651) at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:536) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:490) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:450) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at java.lang.Thread.run(Thread.java:745) [2017-02-04T09:31:45,740][ERROR][o.e.t.n.Netty4Utils ] fatal error on the network layer at org.elasticsearch.transport.netty4.Netty4Utils.maybeDie(Netty4Utils.java:141) at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.exceptionCaught(Netty4HttpRequestHandler.java:76) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelHandlerAdapter.exceptionCaught(ChannelHandlerAdapter.java:78) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.DefaultChannelPipeline$HeadContext.exceptionCaught(DefaultChannelPipeline.java:1301) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.DefaultChannelPipeline.fireExceptionCaught(DefaultChannelPipeline.java:914) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.handleReadException(AbstractNioByteChannel.java:99) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:140) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:651) at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:536) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:490) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:450) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at java.lang.Thread.run(Thread.java:745) [2017-02-04T09:31:45,740][ERROR][o.e.t.n.Netty4Utils ] fatal error on the network layer at org.elasticsearch.transport.netty4.Netty4Utils.maybeDie(Netty4Utils.java:141) at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.exceptionCaught(Netty4HttpRequestHandler.java:76) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.notifyHandlerException(AbstractChannelHandlerContext.java:861) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:375) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:280) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:396) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:248) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:129) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:651) at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:536) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:490) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:450) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at java.lang.Thread.run(Thread.java:745) [2017-02-04T09:31:54,332][WARN ][o.e.m.j.JvmGcMonitorService] [_pF7Uds] [gc][1968] overhead, spent [52s] collecting in the last [52s] [2017-02-04T09:31:54,332][ERROR][o.e.t.n.Netty4Utils ] fatal error on the network layer at org.elasticsearch.transport.netty4.Netty4Utils.maybeDie(Netty4Utils.java:141) at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.exceptionCaught(Netty4HttpRequestHandler.java:76) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.notifyHandlerException(AbstractChannelHandlerContext.java:861) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:375) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:280) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:396) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:248) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:129) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:651) at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:536) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:490) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:450) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at java.lang.Thread.run(Thread.java:745) [2017-02-04T09:31:54,332][ERROR][o.e.t.n.Netty4Utils ] fatal error on the network layer at org.elasticsearch.transport.netty4.Netty4Utils.maybeDie(Netty4Utils.java:141) at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.exceptionCaught(Netty4HttpRequestHandler.java:76) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:275) at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:267) at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131) at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:296) at io.netty.channel.AbstractChannelHandlerContext.notifyHandlerException(AbstractChannelHandlerContext.java:861) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:375) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:280) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:396) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:248) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:351) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:373) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:129) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:651) at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:536) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:490) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:450) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at java.lang.Thread.run(Thread.java:745) [2017-02-04T09:31:54,360][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [] fatal error in thread [elasticsearch[_pF7Uds][bulk][T#1]], exiting java.lang.OutOfMemoryError: Java heap space [2017-02-04T09:31:54,360][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [] fatal error in thread [Thread-39], exiting java.lang.OutOfMemoryError: Java heap space