URGENT PLZ: ISSUE WITH CLIENT THREAD POOL WAITING STATE
Harish Kumar Kandikattu
| DUMP | wait | runnable | total | time |
| 1 | 346 | 135 | 481 | Tue Jul 25 08:07:59 EDT 2017 |
| 2 | 377 | 104 | 481 | Tue Jul 25 08:08:55 EDT 2017 |
| 3 | 372 | 109 | 481 | Tue Jul 25 08:09:54 EDT 2017 |
| 4 | 358 | 123 | 481 | Tue Jul 25 08:10:56 EDT 2017 |
| 5 | 352 | 129 | 481 | Tue Jul 25 08:11:56 EDT 2017 |
| 6 | 370 | 109 | 479 | |
| 7 | 383 | 80 | 463 | Tue Jul 25 08:31:09 EDT 2017 |
| 8 | 380 | 75 | 455 | Tue Jul 25 08:34:52 EDT 2017 |
| 9 | 373 | 36 | 412 | Tue Jul 25 10:40:08 EDT 2017 |
| 10 | 379 | 30 | 409 | Tue Jul 25 11:59:46 EDT 2017 |
| 11 | 377 | 156 | 533 | Tue Jul 25 13:34:12 EDT 2017 |
| 12 | 379 | 154 | 533 | Tue Jul 25 13:39:02 EDT 2017 |
| 13 | 389 | 58 | 447 | Tue Jul 25 14:34:23 EDT 2017 |
| 14 | 405 | 35 | 440 | Tue Jul 25 21:59:47 EDT 2017 |
| 15 | 408 | 42 | 450 | Wed Jul 26 02:55:20 EDT 2017 |
| 16 | 422 | 36 | 458 | Wed Jul 26 05:44:24 EDT 2017 |
Harish Kumar Kandikattu
/*
* This file is part of the QuickServer library
* Copyright (C) QuickServer.org
*
* Use, modification, copying and distribution of this software is subject to
* the terms and conditions of the GNU Lesser General Public License.
* You should have received a copy of the GNU LGP License along with this
* library; if not, you can download a copy from <http://www.quickserver.org/>.
*
* For questions, suggestions, bug-reports, enhancement-requests etc.
* visit http://www.quickserver.org
*
*/
package org.quickserver.util.pool.thread;
import java.util.logging.*;
import java.util.*;
import org.quickserver.util.MyString;
import org.quickserver.net.server.ClientHandler;
import org.quickserver.net.server.ClientEvent;
/**
* This is the worker thread used to handle clients using
* {@link org.quickserver.net.server.ClientHandler}
* @author Akshathkumar Shetty
* @since 1.3
*/
public class ClientThread extends Thread {
private static Logger logger = Logger.getLogger(ClientThread.class.getName());
private static Map idMap = new HashMap();
private String name = "<ClientThread-Pool#";
private ClientPool pool;
private Runnable client;
private int id;
private boolean ready = false;
/**
* Holds the current Thread state. <code><pre>
U = Unknown
S = Started
R - Running a client
I = Idle
L = Looking for client
P = Was sent back to pool
W = Waiting in pool
N = Was notified, Looking for client
D = Dead
</pre></code>
*/
protected volatile char state = 'U';
public boolean isReady() {
return ready;
}
public void clean() {
client = null;
}
public ClientThread(ClientPool pool) {
this(pool, -1);
}
static class InstanceId {
private int id = 0;
public int getNextId() {
return ++id;
}
};
private static int getNewId(int instanceCount) {
InstanceId instanceId = (InstanceId) idMap.get(""+instanceCount);
if(instanceId==null) {
instanceId = new InstanceId();
idMap.put(""+instanceCount, instanceId);
}
return instanceId.getNextId();
}
public ClientThread(ClientPool pool, int instanceCount) {
id = getNewId(instanceCount);
name = name+instanceCount+"-ID:"+id+">";
this.pool = pool;
setName(name);
}
public int getInstanceId() {
return id;
}
private void executeClient() {
boolean niowriteFlag = false;
state = 'R';
if(ClientHandler.class.isInstance(client)) {
niowriteFlag = ((ClientHandler) client).isClientEventNext(ClientEvent.WRITE);
if(niowriteFlag) {
pool.nioWriteStart();
}
} else {
niowriteFlag = false;
}
try {
client.run();
} catch(RuntimeException e) {
logger.warning("RuntimeException @ thread run() : "+getName()+": "+MyString.getStackTrace(e));
} finally {
if(niowriteFlag) {
pool.nioWriteEnd();
}
}
state = 'I';
}
public void run() {
state = 'S';
synchronized(pool) {
if(pool.isClientAvailable()==true) {
ready = true;
pool.notify();
}
}
boolean returnToPool = false;
while(true) {
if(ready) {
state = 'L';
client = pool.getClient();
if(client==null) {
logger.fine("ClientPool returned a null client! Other Thread must have taken my client.. Ok");
} else {
executeClient();
logger.finest("Client returned the thread: "+getName());
client = null;
if(pool==null) {
logger.fine("Could not returning client thread "+getName()+", pool was null!");
state = 'D';
break;
}
}
/*if(pool.isClientAvailable()==true) {
state = 'L';
continue;
}*/
returnToPool = true;
} //end if ready
synchronized(this) {
if(ready==false) ready = true;
if(returnToPool) {
logger.finest("Returning client thread to pool: "+getName());
pool.returnObject(ClientThread.this);
returnToPool = false;
state = 'P';
}
try {
state = 'W';
wait();
state = 'N';
} catch(InterruptedException e) {
logger.finest("Closing thread "+Thread.currentThread().getName()+" since interrupted.");
state = 'D';
break;
}
}
}//end while
}
/**
* Returns the {@link org.quickserver.net.server.ClientHandler} beeing
* run by the ClientThread.
* @since 1.3.1
*/
public Runnable getThread() {
return client;
}
/**
* [ThreadInPool[<Instance Count>]:<id>] - <state> - Client {ClientHandler:...}
* @since 1.4.1
*/
public String toString() {
return super.toString()+" - "+state+" - Client "+client;
}
}
System.out.println(monitorClientThreadPool());
private String monitorClientThreadPool() {
QuickServer server = myServer; // Assign your application's server object reference here
int count = Thread.getAllStackTraces().size();
if (server != null) {
ClientPool clientpool = server.getClientPool();
ObjectPool handlerpool = server.getClientHandlerPool();
ObjectPool datapool = server.getClientDataPool();
ObjectPool bytebufferpool = server.getByteBufferPool();
int max_idle = server.getClientPool().getPoolConfig().getMaxIdle();
int max_active = server.getClientPool().getPoolConfig().getMaxActive();
long client_count=server.getClientCount();
StringBuffer msg = new StringBuffer();
msg.append("{ MonitoringServer : ")
.append("{ ApproxTotalThreads:")
.append(count)
.append(", Max_Idle:")
.append(max_idle)
.append(", Max_Active:")
.append(max_active)
.append(", NumOfClientsConnected:")
.append(client_count)
.append(", [ ");
if (clientpool != null) {
int cp_active = clientpool.getNumActive();
int cp_idle = clientpool.getNumIdle();
msg.append("{ Type:ClientPool, Total:")
.append(cp_active + cp_idle)
.append(", Active:")
.append(cp_active)
.append(", Idle:")
.append(cp_idle)
.append(" }");
}
if (handlerpool != null) {
int h_active = handlerpool.getNumActive();
int h_idle = handlerpool.getNumIdle();
msg.append(", { Type:ClientHandlerPool, Total:")
.append(h_active + h_idle)
.append(", Active:")
.append(h_active)
.append(", Idle:")
.append(h_idle)
.append(" }");
}
if (datapool != null) {
int d_active = datapool.getNumActive();
int d_idle = datapool.getNumIdle();
msg.append(", { Type:ClientDataPool, Total:")
.append(d_active + d_idle)
.append(", Active:")
.append(d_active)
.append(", Idle:")
.append(d_idle)
.append(" }");
}
if (bytebufferpool != null) {
int b_active = bytebufferpool.getNumActive();
int b_idle = bytebufferpool.getNumIdle();
msg.append(", { Type:ByteBufferPool, Total:")
.append(b_active + b_idle)
.append(", Active:")
.append(b_active)
.append(", Idle:")
.append(b_idle)
.append(" }");
}
msg.append(" ] } }");
return msg.toString();
}
return "{ MonitoringServer : NoDataFound }";
}
Steps to fetch data
To get more accurate data, perform steps 1 & 2 with minimal delay in time 20 to 50 seconds should be good.
1. Open log file and search for text "MonitoringServer" and make a note of monitor message. Below is sample monitor message.
''{ MonitoringServer : { ApproxTotalThreads:105, Max_Idle:50, Max_Active:-1, NumOfClientsConnected:11, [ { Type:ClientPool, Total:60, Active:11, Idle:49 }, { Type:ClientHandlerPool, Total:60, Active:11, Idle:49 } ] } }"
2. Get thread dump data
kill -3 <PID of your program>
3. Open thread dump file, select all and copy.
4. Use online free thread dump analyzer tool. https://spotify.github.io/threaddump-analyzer
5. After opening link. Paste the content copied in step3 into text area and click Analyze button. You should see grouping of similar thread stack traces with counts.
Validating fetched data
All below validations indicates that quick server's thread pool behaviour is stable.
1. ClientPool and ClientHandlerPool should always have exact Total,Active and Idle counts.
{ Type:ClientPool, Total:60, Active:11, Idle:49 }
{ Type:ClientHandlerPool, Total:60, Active:11, Idle:49 }
2. Number of clients connected, ClientPool's and ClientHandlerPool's Active count should always be same.
NumOfClientsConnected:11
{ Type:ClientPool, Total:60, Active:11, Idle:49 }
{ Type:ClientHandlerPool, Total:60, Active:11, Idle:49 }
3. ClientPool's and ClientHandlerPool's Idle count should always <=50.
4. Compare ClientPool data and thread dump data. ClientPool's Idle count and awaiting thread count at this line "org.quickserver.util.pool.thread.ClientThread.run(ClientThread.java:166)" should almost match.
Monitor data
{ Type:ClientPool, Total:60, Active:11, Idle:49 }
Data from thread dump analyzer tool.
49 threads with this stack:
"<ClientThread-Pool#1-ID:9>": awaiting notification on [0x00000000b080fe70]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.quickserver.util.pool.thread.ClientThread.run(ClientThread.java:166)
5. ClientPool's Active count and Actively running thread count should almost match.
Monitor data
{ Type:ClientPool, Total:60, Active:11, Idle:49 }
Data from thread dump analyzer tool.
11 threads with this stack:
"<ClientThread-Pool#1-ID:25>": running, holding [0x00000000b19aeed8]
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
at org.quickserver.net.server.impl.BasicClientHandler.readInputStream(BasicClientHandler.java:895)
at org.quickserver.net.server.impl.BlockingClientHandler.readInputStream(BlockingClientHandler.java:484)
at org.quickserver.net.server.impl.BasicClientHandler.readBinary(BasicClientHandler.java:1111)
at org.quickserver.net.server.impl.BlockingClientHandler.processRead(BlockingClientHandler.java:355)
at org.quickserver.net.server.impl.BlockingClientHandler.run(BlockingClientHandler.java:193)
at org.quickserver.util.pool.thread.ClientThread.executeClient(ClientThread.java:107)
at org.quickserver.util.pool.thread.ClientThread.run(ClientThread.java:136)
