Just got reports that our CAS server was down. The JVM PID is still alive, the CPU utilization is throttling around 4% and the server is at 50% memory capacity. Looking at the logs just before the first report came in:
2014-06-04 13:08:37,194 INFO [com.github.inspektr.audit.support.Slf4jLoggingAuditTrailManager] - Audit trail record BEGIN
=============================================================
WHO: someuser
WHAT: ST-129-kFVbjggm7wW6Py1mDVsx-localsso.example.org for http://localhost:9100/myapp/shiro-cas
ACTION: SERVICE_TICKET_CREATED
APPLICATION: CAS
WHEN: Wed Jun 04 13:08:37 EDT 2014
CLIENT IP ADDRESS: 10.10.40.17
SERVER IP ADDRESS: 10.10.41.78
=============================================================
2014-06-04 13:08:37,196 DEBUG [org.jasig.cas.web.flow.TerminateWebSessionListener] - Error getting service from flow state.
java.lang.IllegalStateException: No active FlowSession to access; this FlowExecution has ended
at org.springframework.webflow.engine.impl.FlowExecutionImpl.getActiveSession(FlowExecutionImpl.java:191)
at org.springframework.webflow.engine.impl.RequestControlContextImpl.getFlowScope(RequestControlContextImpl.java:134)
at org.jasig.cas.web.support.WebUtils.getService_aroundBody8(WebUtils.java:87)
at org.jasig.cas.web.support.WebUtils.getService_aroundBody9$advice(WebUtils.java:57)
at org.jasig.cas.web.support.WebUtils.getService(WebUtils.java:1)
at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody0(TerminateWebSessionListener.java:62)
at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody1$advice(TerminateWebSessionListener.java:57)
at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted(TerminateWebSessionListener.java:1)
at org.springframework.webflow.engine.impl.FlowExecutionListeners.fireSessionStarted(FlowExecutionListeners.java:126)
at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:367)
at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:225)
at org.springframework.webflow.executor.FlowExecutorImpl.launchExecution(FlowExecutorImpl.java:140)
at org.springframework.webflow.mvc.servlet.FlowHandlerAdapter.handle(FlowHandlerAdapter.java:193)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody2(SafeDispatcherServlet.java:128)
at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody3$advice(SafeDispatcherServlet.java:57)
at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:1)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at com.github.inspektr.common.web.ClientInfoThreadLocalFilter.doFilter(ClientInfoThreadLocalFilter.java:63)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:929)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1002)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:585)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:310)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-06-04 13:08:37,196 DEBUG [org.jasig.cas.web.flow.TerminateWebSessionListener] - Error getting service from flow state.
java.lang.IllegalStateException: No active FlowSession to access; this FlowExecution has ended
at org.springframework.webflow.engine.impl.FlowExecutionImpl.getActiveSession(FlowExecutionImpl.java:191)
at org.springframework.webflow.engine.impl.RequestControlContextImpl.getFlowScope(RequestControlContextImpl.java:134)
at org.jasig.cas.web.support.WebUtils.getService_aroundBody8(WebUtils.java:87)
at org.jasig.cas.web.support.WebUtils.getService_aroundBody9$advice(WebUtils.java:57)
at org.jasig.cas.web.support.WebUtils.getService(WebUtils.java:1)
at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody0(TerminateWebSessionListener.java:62)
at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody1$advice(TerminateWebSessionListener.java:57)
at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted(TerminateWebSessionListener.java:1)
at org.springframework.webflow.engine.impl.FlowExecutionListeners.fireSessionStarted(FlowExecutionListeners.java:126)
at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:367)
at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:225)
at org.springframework.webflow.executor.FlowExecutorImpl.launchExecution(FlowExecutorImpl.java:140)
at org.springframework.webflow.mvc.servlet.FlowHandlerAdapter.handle(FlowHandlerAdapter.java:193)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody2(SafeDispatcherServlet.java:128)
at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody3$advice(SafeDispatcherServlet.java:57)
at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:1)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at com.github.inspektr.common.web.ClientInfoThreadLocalFilter.doFilter(ClientInfoThreadLocalFilter.java:63)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:929)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1002)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:585)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:310)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Any idea as to what could be going on here? Thanks in advance!
-- You are currently subscribed to cas-...@lists.jasig.org as: jasig-cas-user...@googlegroups.com To unsubscribe, change settings or access archives, see http://www.ja-sig.org/wiki/display/JSG/cas-user