Understanding "libgomp: Thread creation failed: Resource temporarily unavailable" error when fitting multiple SPDE models

[Iavor Bojinov]

Hi all,

I'm trying to fit a large model with multiple SPDE on a cluster with 64 nodes and 256,000 memory. When I fit a small version only using a subset of the data it works perfectly fine, but when ever I try to fit the full thing I am getting the following error

libgomp: 

libgomp: Thread creation failed: Resource temporarily unavailable

libgomp: Thread creation failed: Resource temporarily unavailable

libgomp: Thread creation failed: Resource temporarily unavailable

libgomp: Thread creation failed: Resource temporarily unavailable

libgomp: Thread creation failed: Resource temporarily unavailable

Thread creation failed: Resource temporarily unavailable

libgomp: Error in inla.inlaprogram.has.crashed() : 

Here is the code I am using:

N<-64

inla.setOption("num.threads", N)

points<-cbind(x=data[,2],y=data[,3])

mesh <- inla.mesh.2d(, dom.points, max.edge=c(5, 5), n=4, offset=c(5,5))

# Create the projection matrix

A<-inla.spde.make.A(mesh,loc=points)

A_ARAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[1]))

A_AVAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[2]))

A_CHAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[3]))

A_EVAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[4]))

A_FUAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[5]))

A_LIAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[6]))

A_MCAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[7]))

A_MUAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[8]))

A_NUAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[9]))

A_NCAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[10]))

A_QPAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[11]))

A_RDAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[12]))

A_SOAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[13]))

A_SCAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[14]))

A_SUAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[15]))

A_SWAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[16]))

A_THAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[17]))

A_WBAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[18]))

A_WHAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[19]))

A_WAAtk<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamAtk==all.teams[20]))

A_ARDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[1]))

A_AVDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[2]))

A_CHDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[3]))

A_EVDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[4]))

A_FUDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[5]))

A_LIDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[6]))

A_MCDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[7]))

A_MUDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[8]))

A_NUDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[9]))

A_NCDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[10]))

A_QPDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[11]))

A_RDDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[12]))

A_SODef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[13]))

A_SCDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[14]))

A_SUDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[15]))

A_SWDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[16]))

A_THDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[17]))

A_WBDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[18]))

A_WHDef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[19]))

A_WADef<- inla.spde.make.A(mesh,loc=points,weights=as.numeric(data$TeamDef==all.teams[20]))

# Define a SPDE model using the mesh

spde<- inla.spde2.matern(mesh,alpha=2,theta.prior.mean=c(2.221988,-4.344010),theta.prior.prec=c(1,1)))

# Create a list of indexes for the spatial effect

i_full<-inla.spde.make.index(name="sf_full",n.spde=spde$n.spde)

i_ARAtk<-inla.spde.make.index(name="sf_ARAtk",n.spde=spde$n.spde)

i_AVAtk<-inla.spde.make.index(name="sf_AVAtk",n.spde=spde$n.spde)

i_CHAtk<-inla.spde.make.index(name="sf_CHAtk",n.spde=spde$n.spde)

i_EVAtk<-inla.spde.make.index(name="sf_EVAtk",n.spde=spde$n.spde)

i_FUAtk<-inla.spde.make.index(name="sf_FUAtk",n.spde=spde$n.spde)

i_LIAtk<-inla.spde.make.index(name="sf_LIAtk",n.spde=spde$n.spde)

i_MCAtk<-inla.spde.make.index(name="sf_MCAtk",n.spde=spde$n.spde)

i_MUAtk<-inla.spde.make.index(name="sf_MUAtk",n.spde=spde$n.spde)

i_NUAtk<-inla.spde.make.index(name="sf_NUAtk",n.spde=spde$n.spde)

i_NCAtk<-inla.spde.make.index(name="sf_NCAtk",n.spde=spde$n.spde)

i_QPAtk<-inla.spde.make.index(name="sf_QPAtk",n.spde=spde$n.spde)

i_RDAtk<-inla.spde.make.index(name="sf_RDAtk",n.spde=spde$n.spde)

i_SOAtk<-inla.spde.make.index(name="sf_SOAtk",n.spde=spde$n.spde)

i_SCAtk<-inla.spde.make.index(name="sf_SCAtk",n.spde=spde$n.spde)

i_SUAtk<-inla.spde.make.index(name="sf_SUAtk",n.spde=spde$n.spde)

i_SWAtk<-inla.spde.make.index(name="sf_SWAtk",n.spde=spde$n.spde)

i_THAtk<-inla.spde.make.index(name="sf_THAtk",n.spde=spde$n.spde)

i_WBAtk<-inla.spde.make.index(name="sf_WBAtk",n.spde=spde$n.spde)

i_WHAtk<-inla.spde.make.index(name="sf_WHAtk",n.spde=spde$n.spde)

i_WAAtk<-inla.spde.make.index(name="sf_WAAtk",n.spde=spde$n.spde)

i_ARDef<-inla.spde.make.index(name="sf_ARDef",n.spde=spde$n.spde)

i_AVDef<-inla.spde.make.index(name="sf_AVDef",n.spde=spde$n.spde)

i_CHDef<-inla.spde.make.index(name="sf_CHDef",n.spde=spde$n.spde)

i_EVDef<-inla.spde.make.index(name="sf_EVDef",n.spde=spde$n.spde)

i_FUDef<-inla.spde.make.index(name="sf_FUDef",n.spde=spde$n.spde)

i_LIDef<-inla.spde.make.index(name="sf_LIDef",n.spde=spde$n.spde)

i_MCDef<-inla.spde.make.index(name="sf_MCDef",n.spde=spde$n.spde)

i_MUDef<-inla.spde.make.index(name="sf_MUDef",n.spde=spde$n.spde)

i_NUDef<-inla.spde.make.index(name="sf_NUDef",n.spde=spde$n.spde)

i_NCDef<-inla.spde.make.index(name="sf_NCDef",n.spde=spde$n.spde)

i_QPDef<-inla.spde.make.index(name="sf_QPDef",n.spde=spde$n.spde)

i_RDDef<-inla.spde.make.index(name="sf_RDDef",n.spde=spde$n.spde)

i_SODef<-inla.spde.make.index(name="sf_SODef",n.spde=spde$n.spde)

i_SCDef<-inla.spde.make.index(name="sf_SCDef",n.spde=spde$n.spde)

i_SUDef<-inla.spde.make.index(name="sf_SUDef",n.spde=spde$n.spde)

i_SWDef<-inla.spde.make.index(name="sf_SWDef",n.spde=spde$n.spde)

i_THDef<-inla.spde.make.index(name="sf_THDef",n.spde=spde$n.spde)

i_WBDef<-inla.spde.make.index(name="sf_WBDef",n.spde=spde$n.spde)

i_WHDef<-inla.spde.make.index(name="sf_WHDef",n.spde=spde$n.spde)

i_WADef<-inla.spde.make.index(name="sf_WADef",n.spde=spde$n.spde)

# Stack from the book

stack<-inla.stack(data=list(out=data$Out),A=list(A,A_ARAtk,A_AVAtk,A_CHAtk,A_EVAtk,A_FUAtk,A_LIAtk,A_MCAtk,A_MUAtk,A_NUAtk,A_NCAtk,

A_QPAtk,A_RDAtk,A_SOAtk,A_SCAtk,A_SUAtk,A_SWAtk,A_THAtk,A_WBAtk,A_WHAtk,A_WAAtk,A_ARDef,A_AVDef,A_CHDef,A_EVDef,A_FUDef,A_LIDef,A_MCDef,A_MUDef,A_NUDef,A_NCDef,

A_QPDef,A_RDDef,A_SODef,A_SCDef,A_SUDef,A_SWDef,A_THDef,A_WBDef,A_WHDef,A_WADef, 1),effects=list(i_full,i_ARAtk,i_AVAtk,i_CHAtk,i_EVAtk,i_FUAtk,i_LIAtk,i_MCAtk,i_MUAtk,i_NUAtk,i_NCAtk,

i_QPAtk,i_RDAtk,i_SOAtk,i_SCAtk,i_SUAtk,i_SWAtk,i_THAtk,i_WBAtk,i_WHAtk,i_WAAtk,i_ARDef,i_AVDef,i_CHDef,i_EVDef,i_FUDef,i_LIDef,i_MCDef,i_MUDef,i_NUDef,i_NCDef,

i_QPDef,i_RDDef,i_SODef,i_SCDef,i_SUDef,i_SWDef,i_THDef,i_WBDef,i_WHDef,i_WADef,data.frame(Intercept=rep(1,n))),tag="est")

formula <- out ~ -1+Intercept+f(sf_full,model = spde)+

f(sf_ARAtk,model = spde)+

f(sf_AVAtk,model = spde)+

f(sf_CHAtk,model = spde)+

f(sf_EVAtk,model = spde)+

f(sf_FUAtk,model = spde)+

f(sf_LIAtk,model = spde)+

f(sf_MCAtk,model = spde)+

f(sf_MUAtk,model = spde)+

f(sf_NUAtk,model = spde)+

f(sf_NCAtk,model = spde)+

f(sf_QPAtk,model = spde)+

f(sf_RDAtk,model = spde)+

f(sf_SOAtk,model = spde)+

f(sf_SCAtk,model = spde)+

f(sf_SUAtk,model = spde)+

f(sf_SWAtk,model = spde)+

f(sf_THAtk,model = spde)+

f(sf_WBAtk,model = spde)+

f(sf_WHAtk,model = spde)+

f(sf_WAAtk,model = spde)+

f(sf_ARDef,model = spde)+

f(sf_AVDef,model = spde)+

f(sf_CHDef,model = spde)+

f(sf_EVDef,model = spde)+

f(sf_FUDef,model = spde)+

f(sf_LIDef,model = spde)+

f(sf_MCDef,model = spde)+

f(sf_MUDef,model = spde)+

f(sf_NUDef,model = spde)+

f(sf_NCDef,model = spde)+

f(sf_QPDef,model = spde)+

f(sf_RDDef,model = spde)+

f(sf_SODef,model = spde)+

f(sf_SCDef,model = spde)+

f(sf_SUDef,model = spde)+

f(sf_SWDef,model = spde)+

f(sf_THDef,model = spde)+

f(sf_WBDef,model = spde)+

f(sf_WHDef,model = spde)+

f(sf_WADef,model = spde)

result  <-  inla(formula,family="binomial",data=inla.stack.data(stack),control.predictor=list(A=inla.stack.A(stack)),verbose=TRUE,control.inla=list(int.strategy='eb'),num.threads=N) 

Thanks guys,

Iav 

[Nick Golding]

I've seen this same error running on a cluster with a job scheduler.

In that case the machine had the number of requested cores, but the job submitted was assigned fewer cores than INLA wanted.