MongoDB SocketException Error

[Luca Barbierato]

Dear Derek,

due to the large amount of data stored in MongoDB and the large number of tenants instantiated in SiteWhere, we are experiencing this MongoDB error that stops each connection of SiteWhere to Mongo:

2017-12-07T10:33:28.044+0000 I NETWORK  [conn311] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [127.0.0.1:47894]
2017-12-07T10:33:28.044+0000 I NETWORK  [conn362] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [127.0.0.1:47953]

Our MongoDB configuration is:

# mongod.conf

# for documentation of all options, see:
#   http://docs.mongodb.org/manual/reference/configuration-options/

# Where and how to store data.
storage:
  dbPath: /var/lib/mongodb
  journal:
    enabled: true
#  engine:

#  mmapv1:

#  wiredTiger:

# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  logRotate: rename
  quiet: true
  path: /var/log/mongodb/mongod.log

# network interfaces
net:
  port: 27017
  bindIp: 127.0.0.1

#processManagement:

#security:

#operationProfiling:

#replication:

#sharding:

## Enterprise-Only Options:

#auditLog:

#snmp:

SiteWhere log prints out:

2017-12-07 10:35:37,420 ERROR [AssignmentStateManager]  Unhandled exeception while updating assignment state.
com.mongodb.MongoTimeoutException: Timeout waiting for a pooled item after 120000 MILLISECONDS
        at com.mongodb.ConcurrentPool.get(ConcurrentPool.java:113)
        at com.mongodb.PooledConnectionProvider.get(PooledConnectionProvider.java:75)
        at com.mongodb.DefaultServer.getConnection(DefaultServer.java:61)
        at com.mongodb.BaseCluster$WrappedServer.getConnection(BaseCluster.java:254)
        at com.mongodb.DBTCPConnector$MyPort.getConnection(DBTCPConnector.java:513)
        at com.mongodb.DBTCPConnector$MyPort.get(DBTCPConnector.java:456)
        at com.mongodb.DBTCPConnector.innerCall(DBTCPConnector.java:288)
        at com.mongodb.DBTCPConnector.call(DBTCPConnector.java:273)
        at com.mongodb.DBCollectionImpl.find(DBCollectionImpl.java:84)
        at com.mongodb.DBCollectionImpl.find(DBCollectionImpl.java:66)
        at com.mongodb.DBCollection.findOne(DBCollection.java:940)
        at com.mongodb.DBCollection.findOne(DBCollection.java:914)
        at com.mongodb.DBCollection.findOne(DBCollection.java:858)
        at com.sitewhere.mongodb.device.MongoDeviceManagement.getDeviceAssignmentDBObjectByToken(MongoDeviceManagement.java:1043)
        at com.sitewhere.mongodb.device.MongoDeviceManagement.assertDeviceAssignment(MongoDeviceManagement.java:1735)
        at com.sitewhere.mongodb.device.MongoDeviceManagement.updateDeviceAssignmentState(MongoDeviceManagement.java:866)
        at com.sitewhere.device.DeviceManagementDecorator.updateDeviceAssignmentState(DeviceManagementDecorator.java:229)
        at com.sitewhere.device.AssignmentStateManager$Persistence.run(AssignmentStateManager.java:195)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
2017-12-07 10:35:37,616 ERROR [DeviceEventBuffer]  Connection to MongoDB lost.
com.mongodb.MongoTimeoutException: Timeout waiting for a pooled item after 120000 MILLISECONDS
        at com.mongodb.ConcurrentPool.get(ConcurrentPool.java:113)
        at com.mongodb.PooledConnectionProvider.get(PooledConnectionProvider.java:75)
        at com.mongodb.DefaultServer.getConnection(DefaultServer.java:61)
        at com.mongodb.BaseCluster$WrappedServer.getConnection(BaseCluster.java:254)
        at com.mongodb.DBTCPConnector$MyPort.getConnection(DBTCPConnector.java:513)
        at com.mongodb.DBTCPConnector$MyPort.get(DBTCPConnector.java:456)
        at com.mongodb.DBTCPConnector.getPrimaryPort(DBTCPConnector.java:415)
        at com.mongodb.DBCollectionImpl.executeBulkWriteOperation(DBCollectionImpl.java:144)
        at com.mongodb.DBCollection.executeBulkWriteOperation(DBCollection.java:1737)
        at com.mongodb.DBCollection.executeBulkWriteOperation(DBCollection.java:1733)
        at com.mongodb.BulkWriteOperation.execute(BulkWriteOperation.java:93)
        at com.sitewhere.mongodb.device.DeviceEventBuffer$EventSender.run(DeviceEventBuffer.java:121)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
2017-12-07 10:37:37,424 ERROR [AssignmentStateManager]  Unhandled exeception while updating assignment state.
com.mongodb.MongoTimeoutException: Timeout waiting for a pooled item after 120000 MILLISECONDS
        at com.mongodb.ConcurrentPool.get(ConcurrentPool.java:113)
        at com.mongodb.PooledConnectionProvider.get(PooledConnectionProvider.java:75)
        at com.mongodb.DefaultServer.getConnection(DefaultServer.java:61)
        at com.mongodb.BaseCluster$WrappedServer.getConnection(BaseCluster.java:254)
        at com.mongodb.DBTCPConnector$MyPort.getConnection(DBTCPConnector.java:513)
        at com.mongodb.DBTCPConnector$MyPort.get(DBTCPConnector.java:456)
        at com.mongodb.DBTCPConnector.innerCall(DBTCPConnector.java:288)
        at com.mongodb.DBTCPConnector.call(DBTCPConnector.java:273)
        at com.mongodb.DBCollectionImpl.find(DBCollectionImpl.java:84)
        at com.mongodb.DBCollectionImpl.find(DBCollectionImpl.java:66)
        at com.mongodb.DBCollection.findOne(DBCollection.java:940)
        at com.mongodb.DBCollection.findOne(DBCollection.java:914)
        at com.mongodb.DBCollection.findOne(DBCollection.java:858)
        at com.sitewhere.mongodb.device.MongoDeviceManagement.getDeviceAssignmentDBObjectByToken(MongoDeviceManagement.java:1043)
        at com.sitewhere.mongodb.device.MongoDeviceManagement.assertDeviceAssignment(MongoDeviceManagement.java:1735)
        at com.sitewhere.mongodb.device.MongoDeviceManagement.updateDeviceAssignmentState(MongoDeviceManagement.java:866)
        at com.sitewhere.device.DeviceManagementDecorator.updateDeviceAssignmentState(DeviceManagementDecorator.java:229)
        at com.sitewhere.device.AssignmentStateManager$Persistence.run(AssignmentStateManager.java:195)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

Is there a special configuration to run MongoDB in a better way to manage SiteWhere interactions? Sorry but we haven't experience in DB management..

Kind regards,

Luca