Hi,
I am using undertow-core 2.2.22.Final with jboss-threads 3.5.0.Final and Java11.
I noticed that during the restart of the server (thousands of connections will attempt to connect from many other 3rd party servers on startup and also we cache many files on startup), sometimes the process hangs. I tried to take multiple thread dumps which also took a very long time.
The thread dumps are showing that one of the workers is holding the lock. Is there a way to solve it?
Thread dump 1:
"XNIO-1 task-219" prio=5 Id=381 BLOCKED on java.lang.Object@2896e999 owned by "XNIO-1 task-220" Id=382
at app//org.xnio.nio.WorkerThread.execute(WorkerThread.java:622)
at app//org.xnio.nio.NioTcpServerHandle.channelClosed(NioTcpServerHandle.java:111)
at app//org.xnio.nio.NioSocketStreamConnection.closeAction(NioSocketStreamConnection.java:139)
at app//org.xnio.Connection.close(Connection.java:132)
at app//io.undertow.server.AbstractServerConnection.close(AbstractServerConnection.java:159)
at app//org.xnio.IoUtils.safeClose(IoUtils.java:152)
at app//io.undertow.server.protocol.http.HttpResponseConduit.flush(HttpResponseConduit.java:757)
at app//io.undertow.conduits.HeadStreamSinkConduit.flush(HeadStreamSinkConduit.java:131)
at app//org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162)
at app//io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119)
at app//io.undertow.io.AsyncSenderImpl.close(AsyncSenderImpl.java:376)
at app//io.undertow.io.DefaultIoCallback.onComplete(DefaultIoCallback.java:54)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:331)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:316)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:354)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:340)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:345)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.postProcess(RequestHandler.java:815)
at app//io.test.platform.handler.RequestHandler$RequestProcessor$2.lambda$onSuccess$0(RequestHandler.java:671)
at app//org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at app//org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at app//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1522)
at org.xnio.XnioWorker$WorkerThreadFactory$1$1.run(XnioWorker.java:1282)
at java...@11.0.17/java.lang.Thread.run(Thread.java:839)
"XNIO-1 task-220" prio=5 Id=382 RUNNABLE
at app//org.jboss.logging.Logger.logf(Logger.java:2397)
at app//org.jboss.logging.DelegatingBasicLogger.tracef(DelegatingBasicLogger.java:124)
at app//org.xnio.nio.WorkerThread.execute(WorkerThread.java:624)
- locked java.lang.Object@2896e999
at app//org.xnio.nio.NioTcpServerHandle.channelClosed(NioTcpServerHandle.java:111)
at app//org.xnio.nio.NioSocketStreamConnection.closeAction(NioSocketStreamConnection.java:139)
at app//org.xnio.Connection.close(Connection.java:132)
at app//io.undertow.server.AbstractServerConnection.close(AbstractServerConnection.java:159)
at app//org.xnio.IoUtils.safeClose(IoUtils.java:152)
at app//io.undertow.server.protocol.http.HttpResponseConduit.flush(HttpResponseConduit.java:757)
at app//io.undertow.conduits.HeadStreamSinkConduit.flush(HeadStreamSinkConduit.java:131)
at app//org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162)
at app//io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119)
at app//io.undertow.io.AsyncSenderImpl.close(AsyncSenderImpl.java:376)
at app//io.undertow.io.DefaultIoCallback.onComplete(DefaultIoCallback.java:54)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:331)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:316)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:354)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:340)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:345)
at app//io.test.platform.handler.RequestHandler$RequestProcessor$1.lambda$onFailure$1(RequestHandler.java:555)
at app//org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at app//org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at app//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1522)
at org.xnio.XnioWorker$WorkerThreadFactory$1$1.run(XnioWorker.java:1282)
at java...@11.0.17/java.lang.Thread.run(Thread.java:839)
"XNIO-1 task-227" prio=5 Id=389 BLOCKED on java.lang.Object@2896e999 owned by "XNIO-1 task-220" Id=382
at app//org.xnio.nio.WorkerThread.execute(WorkerThread.java:622)
at app//org.xnio.nio.NioTcpServerHandle.channelClosed(NioTcpServerHandle.java:111)
at app//org.xnio.nio.NioSocketStreamConnection.closeAction(NioSocketStreamConnection.java:139)
at app//org.xnio.Connection.close(Connection.java:132)
at app//io.undertow.server.AbstractServerConnection.close(AbstractServerConnection.java:159)
at app//org.xnio.IoUtils.safeClose(IoUtils.java:152)
at app//io.undertow.server.protocol.http.HttpResponseConduit.flush(HttpResponseConduit.java:757)
at app//io.undertow.conduits.HeadStreamSinkConduit.flush(HeadStreamSinkConduit.java:131)
at app//org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162)
at app//io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119)
at app//io.undertow.io.AsyncSenderImpl.close(AsyncSenderImpl.java:376)
at app//io.undertow.io.DefaultIoCallback.onComplete(DefaultIoCallback.java:54)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:331)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:316)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:354)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:340)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:345)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.postProcess(RequestHandler.java:815)
at app//io.test.platform.handler.RequestHandler$RequestProcessor$2.lambda$onSuccess$0(RequestHandler.java:671)
at app//org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at app//org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at app//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1522)
at org.xnio.XnioWorker$WorkerThreadFactory$1$1.run(XnioWorker.java:1282)
at java...@11.0.17/java.lang.Thread.run(Thread.java:839)
Thread dump 2:
"XNIO-1 task-7" prio=5 Id=133 BLOCKED on java.lang.Object@2896e999 owned by "XNIO-1 task-9" Id=135
at app//org.xnio.nio.WorkerThread.execute(WorkerThread.java:622)
at app//org.xnio.nio.NioTcpServerHandle.channelClosed(NioTcpServerHandle.java:111)
at app//org.xnio.nio.NioSocketStreamConnection.closeAction(NioSocketStreamConnection.java:139)
at app//org.xnio.Connection.close(Connection.java:132)
at app//io.undertow.server.AbstractServerConnection.close(AbstractServerConnection.java:159)
at app//org.xnio.IoUtils.safeClose(IoUtils.java:152)
at app//io.undertow.server.protocol.http.HttpResponseConduit.flush(HttpResponseConduit.java:757)
at app//io.undertow.conduits.HeadStreamSinkConduit.flush(HeadStreamSinkConduit.java:131)
at app//org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162)
at app//io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119)
at app//io.undertow.io.AsyncSenderImpl.close(AsyncSenderImpl.java:376)
at app//io.undertow.io.DefaultIoCallback.onComplete(DefaultIoCallback.java:54)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:331)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:316)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:354)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:340)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:345)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.process(RequestHandler.java:419)
at app//io.test.platform.handler.RequestHandler.lambda$handleRequest$1(RequestHandler.java:255)
at app//org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at app//org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at app//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1522)
at org.xnio.XnioWorker$WorkerThreadFactory$1$1.run(XnioWorker.java:1282)
at java...@11.0.17/java.lang.Thread.run(Thread.java:839)
"XNIO-1 task-8" prio=5 Id=134 BLOCKED on java.lang.Object@2896e999 owned by "XNIO-1 task-9" Id=135
at app//org.xnio.nio.WorkerThread.execute(WorkerThread.java:622)
at app//org.xnio.nio.NioTcpServerHandle.channelClosed(NioTcpServerHandle.java:111)
at app//org.xnio.nio.NioSocketStreamConnection.closeAction(NioSocketStreamConnection.java:139)
at app//org.xnio.Connection.close(Connection.java:132)
at app//io.undertow.server.AbstractServerConnection.close(AbstractServerConnection.java:159)
at app//org.xnio.IoUtils.safeClose(IoUtils.java:152)
at app//io.undertow.server.protocol.http.HttpResponseConduit.flush(HttpResponseConduit.java:757)
at app//io.undertow.conduits.HeadStreamSinkConduit.flush(HeadStreamSinkConduit.java:131)
at app//org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162)
at app//io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119)
at app//io.undertow.io.AsyncSenderImpl.close(AsyncSenderImpl.java:376)
at app//io.undertow.io.DefaultIoCallback.onComplete(DefaultIoCallback.java:54)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:331)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:316)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:354)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:340)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:345)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.process(RequestHandler.java:419)
at app//io.test.platform.handler.RequestHandler.lambda$handleRequest$1(RequestHandler.java:255)
at app//org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at app//org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at app//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1522)
at org.xnio.XnioWorker$WorkerThreadFactory$1$1.run(XnioWorker.java:1282)
at java...@11.0.17/java.lang.Thread.run(Thread.java:839)
"XNIO-1 task-9" prio=5 Id=135 RUNNABLE
at app//org.xnio.nio.WorkerThread.execute(WorkerThread.java:623)
- locked java.lang.Object@2896e999
at app//org.xnio.nio.NioTcpServerHandle.channelClosed(NioTcpServerHandle.java:111)
at app//org.xnio.nio.NioSocketStreamConnection.closeAction(NioSocketStreamConnection.java:139)
at app//org.xnio.Connection.close(Connection.java:132)
at app//io.undertow.server.AbstractServerConnection.close(AbstractServerConnection.java:159)
at app//org.xnio.IoUtils.safeClose(IoUtils.java:152)
at app//io.undertow.server.protocol.http.HttpResponseConduit.flush(HttpResponseConduit.java:757)
at app//io.undertow.conduits.HeadStreamSinkConduit.flush(HeadStreamSinkConduit.java:131)
at app//org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162)
at app//io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119)
at app//io.undertow.io.AsyncSenderImpl.close(AsyncSenderImpl.java:376)
at app//io.undertow.io.DefaultIoCallback.onComplete(DefaultIoCallback.java:54)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:331)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:316)
at app//io.undertow.io.AsyncSenderImpl.send(AsyncSenderImpl.java:354)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:340)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.sendErrorResponse(RequestHandler.java:345)
at app//io.test.platform.handler.RequestHandler$RequestProcessor.process(RequestHandler.java:419)
at app//io.test.platform.handler.RequestHandler.lambda$handleRequest$1(RequestHandler.java:255)
at app//org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at app//org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at app//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1522)
at org.xnio.XnioWorker$WorkerThreadFactory$1$1.run(XnioWorker.java:1282)
at java...@11.0.17/java.lang.Thread.run(Thread.java:839)
I have also attached both the thread dump.