Jenkins have been running 2.26 without issues for a few weeks. Recently started seeing failures like this.

[Rinaldo DiGiorgio]

Timeouts are not working, we have a timeout for  4 hours, it seems to not work. we end up with an unusable system nothing can be done from the ui, a restart is required. Then when it restarts it tries to contiue the jobs and locks up again. 

Thread #234
	at DSL.sh(completed process (code 0) in /scratch/mach5-j/jenkins/jobs/9-dev/workspace@tmp/durable-49708009 (pid: 0))
	at elastic.pushResultsToElastic(file:/scratch/mach5-j/jenkins/jobs/9-dev/builds/1522/libs/mach5PipelineScripts/src/elastic.groovy:94)
	at pipeline.startBuildTest(/scratch/mach5-j/jenkins/jobs/9-dev/builds/1522/libs/mach5PipelineScripts/src/pipeline.groovy:260)
	at DSL.node(running on )
	at pipeline.startBuildTest(/scratch/mach5-j/jenkins/jobs/9-dev/builds/1522/libs/mach5PipelineScripts/src/pipeline.groovy:255)
	at DSL.timeout(Native Method)
	at pipeline.startBuildTest(/scratch/mach5-j/jenkins/jobs/9-dev/builds/1522/libs/mach5PipelineScripts/src/pipeline.groovy:248)
	at DSL.timestamps(Native Method)
	at pipeline.startBuildTest(/scratch/mach5-j/jenkins/jobs/9-dev/builds/1522/libs/mach5PipelineScripts/src/pipeline.groovy:2)
	at WorkflowScript.run(WorkflowScript:2)

Thread #18.parallel(Native Method)
    at DSL.parallel(Native Method)ratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:251)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:207)
    at DSL.parallel(Native Method)ratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:248)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:251)
    at DSL.timeout(Native Method)cratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:2)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:248)
    at DSL.timestamps(Native Method)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:2)rc 128P-256G) (pid: 41569))
    at WorkflowScript.run(WorkflowScript:2)e/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:105)
Thread #286estPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.sh(completed process (code 0) in /scratch/jenkins/workspace/9-dev/test@tmp/durable-8ae39673 on host08giy (Solaris 11.1 Sparc 128P-256G) (pid: 41569))
    at util.shcmd(file:/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:105)n.groovy:175)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.dir(Native Method)ratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:175)
    at DSL.node(running on host08giy (Solaris 11.1 Sparc 128P-256G))
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:140)
    at DSL.parallel(Native Method)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:207)
Thread #278
    at DSL.bat(completed process (code 0) in c:\jenkins\workspace\9-dev\test@tmp\durable-40672e24 on hostag037 (Windows 2012 R2 24P-144G))
    at util.shcmd(file:/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:103)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.dir(Native Method)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:175)
    at DSL.node(running on hostag037 (Windows 2012 R2 24P-144G))
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:140)
    at DSL.parallel(Native Method)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:207)
Thread #284
    at DSL.sh(completed process (code 0) in /scratch/jenkins/workspace/9-dev/test@tmp/durable-5c8c6290 on sca00fxd (Solaris 11.2  x86 32P-256G host121) (pid: 8410))
    at util.shcmd(file:/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:105)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.dir(Native Method)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:175)
    at DSL.node(running on sca00fxd (Solaris 11.2  x86 32P-256G host121))
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:149)
Thread #222
    at DSL.sh(completed process (code 0) in /scratch/jenkins/workspace/9-dev/test@tmp/durable-dde6dbe5 on host269 (Solaris 11.2 Sparc 128P-256G) (pid: 14529))
    at util.shcmd(file:/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:105)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.dir(Native Method)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:175)
    at DSL.node(running on host269 (Solaris 11.2 Sparc 128P-256G))
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:149)
Thread #280
    at DSL.sh(completed process (code 0) in /scratch/jenkins/workspace/9-dev/test@tmp/durable-a3f59316 on host262 (Solaris 11.1 Sparc 128P-256G) (pid: 29906))
    at util.shcmd(file:/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:105)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.dir(Native Method)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:175)
    at DSL.node(running on host262 (Solaris 11.1 Sparc 128P-256G))
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:158)
Thread #276
    at DSL.bat(completed process (code 0) in c:\jenkins\workspace\9-dev\test@tmp\durable-ec1b552f on hostag045 (Windows 2012 R2 24P-144G))
    at util.shcmd(file:/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/util.groovy:103)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:177)
    at DSL.dir(Native Method)
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:175)
    at DSL.node(running on hostag045 (Windows 2012 R2 24P-144G))
    at jdkTestPlugin.call(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/vars/jdkTestPlugin.groovy:9)
    at pipeline.startBuildTest(/scratch/internalname/jenkins/jobs/9-dev/builds/1526/libs/mach5PipelineScripts/src/pipeline.groovy:149)