This probably needs code review, but fingers crossed there are some ideas out there!
1/ When an RTMP connection closed, a bug in my code caused an exception to be thrown in org.red5.server.BaseConnection.close->org.red5.server.Scope.disconnect
2/ The consequence of this was that other RTMP connections became unable to return data, and my server went down.
*** Stack trace for exception caused by me ***
2011-06-08 19:04:30,264 [ERROR] o.w.networking.starburst.Starburst - Unexpected exception in onRed5Disconnect
__exception_stack_trace_start
java.lang.ArrayIndexOutOfBoundsException
at sun.security.provider.DigestBase.engineUpdate(DigestBase.java:110)
at sun.security.provider.MD5.implDigest(MD5.java:88)
at sun.security.provider.DigestBase.engineDigest(DigestBase.java:169)
at sun.security.provider.DigestBase.engineDigest(DigestBase.java:148)
at java.security.MessageDigest$Delegate.engineDigest(MessageDigest.java:546)
at java.security.MessageDigest.digest(MessageDigest.java:323)
at org.scale7.networking.clustering.ClusterNodesHash$HashCalculator.hash(ClusterNodesHash.java:131)
at org.scale7.networking.clustering.ClusterNodesHash.getNodeByResponsibility(ClusterNodesHash.java:98)
at org.wyki.networking.starburst.ClusterNodesManager.getGatewayNodeForEvent(ClusterNodesManager.java:106)
at org.wyki.networking.starburst.EventRoutingManager.forwardEventToEventPathGateways(EventRoutingManager.java:462)
at org.wyki.networking.starburst.EventRoutingManager.notifyEventListeners(EventRoutingManager.java:214)
at org.wyki.networking.starburst.Starburst.notifyEvent(Starburst.java:189)
at com.fightmymonster.game.GameConnectionWatcher.onDisconnect(GameConnectionWatcher.java:27)
at org.wyki.networking.starburst.Starburst.onRed5Disconnect(Starburst.java:461)
at org.wyki.networking.starburst.StarburstRed5App.disconnect(StarburstRed5App.java:119)
at com.fightmymonster.Application.disconnect(Application.java:325)
at org.red5.server.Scope.disconnect(Scope.java:440)
at org.red5.server.BaseConnection.close(BaseConnection.java:395)
at org.red5.server.net.rtmp.RTMPConnection.close(RTMPConnection.java:620)
at org.red5.server.net.rtmp.RTMPMinaConnection.close(RTMPMinaConnection.java:81)
at org.red5.server.net.rtmp.RTMPMinaConnection.onInactive(RTMPMinaConnection.java:237)
at org.red5.server.net.rtmp.RTMPConnection$KeepAliveJob.execute(RTMPConnection.java:1181)
at org.red5.server.scheduling.QuartzSchedulingServiceJob.execute(QuartzSchedulingServiceJob.java:59)
at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:525)
__exception_stack_trace_stop
*** Example stack traces for following exceptions *not* caused by me ***
2011-06-08 19:04:31,274 [ERROR] o.w.networking.starburst.Starburst - Unexpected exception in onRed5Disconnect
__exception_stack_trace_start
java.lang.NullPointerException
at org.red5.server.net.rtmp.RTMPConnection.registerPendingCall(RTMPConnection.java:798)
at org.red5.server.net.rtmp.RTMPConnection.invoke(RTMPConnection.java:808)
at org.red5.server.net.rtmp.RTMPConnection.invoke(RTMPConnection.java:779)
at org.red5.server.net.rtmp.RTMPConnection.invoke(RTMPConnection.java:834)
at org.wyki.networking.starburst.UserContext.flushMessages(UserContext.java:420)
at org.wyki.networking.starburst.UserContext.send(UserContext.java:310)
at org.wyki.networking.starburst.EventRoutingManager.notifyLocalListenersOnPath(EventRoutingManager.java:567)
at org.wyki.networking.starburst.EventRoutingManager.notifyLocalListeners(EventRoutingManager.java:491)
at org.wyki.networking.starburst.EventRoutingManager.notifyEventListeners(EventRoutingManager.java:217)
at org.wyki.networking.starburst.Starburst.notifyEvent(Starburst.java:189)
at com.fightmymonster.game.GameConnectionWatcher.onDisconnect(GameConnectionWatcher.java:27)
at org.wyki.networking.starburst.Starburst.onRed5Disconnect(Starburst.java:461)
at org.wyki.networking.starburst.StarburstRed5App.disconnect(StarburstRed5App.java:119)
at com.fightmymonster.Application.disconnect(Application.java:325)
at org.red5.server.Scope.disconnect(Scope.java:440)
at org.red5.server.BaseConnection.close(BaseConnection.java:395)
at org.red5.server.net.rtmp.RTMPConnection.close(RTMPConnection.java:620)
at org.red5.server.net.rtmp.RTMPMinaConnection.close(RTMPMinaConnection.java:81)
at org.red5.server.net.rtmp.RTMPMinaConnection.onInactive(RTMPMinaConnection.java:237)
at org.red5.server.net.rtmp.RTMPConnection$KeepAliveJob.execute(RTMPConnection.java:1181)
at org.red5.server.scheduling.QuartzSchedulingServiceJob.execute(QuartzSchedulingServiceJob.java:59)
at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:525)
__exception_stack_trace_stop
1-06-08 19:06:15,148 [ERROR] com.fightmymonster.npc.NpcResponder - class com.fightmymonster.rmi.game.MakeFightChallenge.doWork() threw an exception
__exception_stack_trace_start
java.lang.NullPointerException
at org.red5.server.net.rtmp.RTMPConnection.registerPendingCall(RTMPConnection.java:798)
at org.red5.server.net.rtmp.RTMPConnection.invoke(RTMPConnection.java:808)
at org.red5.server.net.rtmp.RTMPConnection.invoke(RTMPConnection.java:779)
at org.red5.server.net.rtmp.RTMPConnection.invoke(RTMPConnection.java:834)
at org.wyki.networking.starburst.UserContext.flushMessages(UserContext.java:420)
at org.wyki.networking.starburst.UserContext.send(UserContext.java:310)
at org.wyki.networking.starburst.Starburst.deliverMessage(Starburst.java:682)
at org.wyki.networking.starburst.Starburst.sendMessage(Starburst.java:160)
at com.fightmymonster.game.rumbles.Rumble.doSaveSingle(Rumble.java:183)
at com.fightmymonster.game.rumbles.MonsterRumble.doSaveSingle(MonsterRumble.java:30)
at com.fightmymonster.game.rumbles.ChallengeRumble.doSaveSingle(ChallengeRumble.java:74)
at com.fightmymonster.game.rumbles.Rumble.savePaired(Rumble.java:152)
at com.fightmymonster.rmi.game.MakeFightChallenge.doWork(MakeFightChallenge.java:174)
at com.fightmymonster.npc.NpcResponder.runOperation(NpcResponder.java:72)
at com.fightmymonster.npc.NpcInitiator.process(NpcInitiator.java:393)
at org.wyki.networking.starburst.UserContextManager.enumerateUserContexts(UserContextManager.java:77)
at org.wyki.networking.starburst.Starburst.enumerateUserContexts(Starburst.java:243)
at com.fightmymonster.npc.NpcInitiator$2.run(NpcInitiator.java:96)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
__exception_stack_trace_stop
Best, Dominic