27 views

Skip to first unread message

Aug 2, 2024, 12:46:05 PMAug 2

to nimble-users

Dear all,

This is to report an issue that I have using Nimble (version 1.2.1) under Linux but not under Windows: this is related to one of the examples on how to pertforme extra calculations with Nimble using the package runMCMCbtadjust: https://cran.r-project.org/web/packages/runMCMCbtadjust/vignettes/runMCMCbtadjust_extraCalculations.html

The code is below:

library(nimble)

library(runMCMCbtadjust)

library(runMCMCbtadjust)

set.seed(1)

y1000<-rnorm(n=1000,mean=600,sd=30)

ModelData <-list(mass = y1000)

ModelConsts <- list(nobs = length(y1000))

ModelCode<-nimbleCode(

{

# Priors

population.mean ~ dunif(0,5000)

#population.mean ~ dnorm(0,sd=100)

population.sd ~ dunif(0,100)

# Normal distribution parameterized by precision = 1/variance in Nimble

population.variance <- population.sd * population.sd

precision <- 1 / population.variance

# Likelihood

for(i in 1:nobs){

mass[i] ~ dnorm(population.mean, precision)

}

})

ModelInits <- function()

{list (population.mean = rnorm(1,600,90), population.sd = runif(1, 1, 30))}

Nchains <- 3

set.seed(1)

Inits<-lapply(1:Nchains,function(x){ModelInits()})

#specifying the names of parameters to analyse and save:

params <- c("population.mean", "population.sd")

calculations.for.modalDIC<-expression(

{

Model1.EC<-Model[[1]]

## first preparing the sampled parameters in a matrix format; uses the as.matrix function specific to mcmc.list objects; also stocking names of the parameters

samples.List.matrix.EC<-as.matrix(samplesList.temp)

names.samples.EC<-dimnames(samples.List.matrix.EC)[[2]]

## second preparing the names of variables to calculate on:

varNames.EC<-Model1.EC$getVarNames()

DatavarNames.EC<-names(data)

notDatavarNames.EC<-setdiff(varNames.EC,DatavarNames.EC)

## third writing and compiling the nimbleFunction we will use:

logProbCalc.EC <- nimbleFunction(

setup = function(model,names.ref.list,notDatavarNames,DatavarNames) {

},

run = function(P = double(1)) { ##NB: double(1) means this if of double type and has one dimension

values(model,names.ref.list) <<- P

model$calculate(notDatavarNames)

return(model$calculate(DatavarNames))

returnType(double(0))

})

logProbCalcPrepared.EC <- logProbCalc.EC(Model1.EC, names.samples.EC, notDatavarNames.EC, DatavarNames.EC)

ClogProbCalcPrepared.EC <- compileNimble(Model1.EC, logProbCalcPrepared.EC)

## fourth, running through all the samples in a lapply function:

logLiks.EC<-sapply(1:(dim(samples.List.matrix.EC)[1]),function(toto)

{

ClogProbCalcPrepared.EC$logProbCalcPrepared.EC$run(samples.List.matrix.EC[toto,])

})

#mode type DIC; cf. Celeux et al. 2006 Bayesian analysis

DIC.mode.EC<--4*mean(logLiks.EC)+2*max(logLiks.EC)

p.DIC.mode.EC<--2*mean(logLiks.EC)+2*max(logLiks.EC)

#calculation of classical DIC; cf. Celeux et al. 2006 Bayesian analysis

logLiks.meanparams.EC<-ClogProbCalcPrepared.EC$logProbCalcPrepared.EC$run(colMeans(samples.List.matrix.EC))

DIC.EC<--4*mean(logLiks.EC)+2*logLiks.meanparams.EC

p.DIC.EC<--2*mean(logLiks.EC)+2*logLiks.meanparams.EC

list(DIC.mode=DIC.mode.EC,p.DIC.mode=p.DIC.mode.EC,DIC=DIC.EC,p.DIC=p.DIC.EC)

}

)

out.mcmc<-runMCMC_btadjust(code=ModelCode, constants = ModelConsts, data = ModelData, MCMC_language="Nimble",

Nchains=Nchains, params=params, inits=Inits,

niter.min=1000, niter.max=300000,

nburnin.min=100, nburnin.max=200000,

thin.min=1, thin.max=1000,

conv.max=1.05, neff.min=1000,

control=list(print.diagnostics=FALSE),

control.MCMC=list(includeParentNodes=TRUE,extraCalculations=calculations.for.modalDIC))

attributes(out.mcmc)<-c(attributes(out.mcmc),DIC=NA)

y1000<-rnorm(n=1000,mean=600,sd=30)

ModelData <-list(mass = y1000)

ModelConsts <- list(nobs = length(y1000))

ModelCode<-nimbleCode(

{

# Priors

population.mean ~ dunif(0,5000)

#population.mean ~ dnorm(0,sd=100)

population.sd ~ dunif(0,100)

# Normal distribution parameterized by precision = 1/variance in Nimble

population.variance <- population.sd * population.sd

precision <- 1 / population.variance

# Likelihood

for(i in 1:nobs){

mass[i] ~ dnorm(population.mean, precision)

}

})

ModelInits <- function()

{list (population.mean = rnorm(1,600,90), population.sd = runif(1, 1, 30))}

Nchains <- 3

set.seed(1)

Inits<-lapply(1:Nchains,function(x){ModelInits()})

#specifying the names of parameters to analyse and save:

params <- c("population.mean", "population.sd")

calculations.for.modalDIC<-expression(

{

Model1.EC<-Model[[1]]

## first preparing the sampled parameters in a matrix format; uses the as.matrix function specific to mcmc.list objects; also stocking names of the parameters

samples.List.matrix.EC<-as.matrix(samplesList.temp)

names.samples.EC<-dimnames(samples.List.matrix.EC)[[2]]

## second preparing the names of variables to calculate on:

varNames.EC<-Model1.EC$getVarNames()

DatavarNames.EC<-names(data)

notDatavarNames.EC<-setdiff(varNames.EC,DatavarNames.EC)

## third writing and compiling the nimbleFunction we will use:

logProbCalc.EC <- nimbleFunction(

setup = function(model,names.ref.list,notDatavarNames,DatavarNames) {

},

run = function(P = double(1)) { ##NB: double(1) means this if of double type and has one dimension

values(model,names.ref.list) <<- P

model$calculate(notDatavarNames)

return(model$calculate(DatavarNames))

returnType(double(0))

})

logProbCalcPrepared.EC <- logProbCalc.EC(Model1.EC, names.samples.EC, notDatavarNames.EC, DatavarNames.EC)

ClogProbCalcPrepared.EC <- compileNimble(Model1.EC, logProbCalcPrepared.EC)

## fourth, running through all the samples in a lapply function:

logLiks.EC<-sapply(1:(dim(samples.List.matrix.EC)[1]),function(toto)

{

ClogProbCalcPrepared.EC$logProbCalcPrepared.EC$run(samples.List.matrix.EC[toto,])

})

#mode type DIC; cf. Celeux et al. 2006 Bayesian analysis

DIC.mode.EC<--4*mean(logLiks.EC)+2*max(logLiks.EC)

p.DIC.mode.EC<--2*mean(logLiks.EC)+2*max(logLiks.EC)

#calculation of classical DIC; cf. Celeux et al. 2006 Bayesian analysis

logLiks.meanparams.EC<-ClogProbCalcPrepared.EC$logProbCalcPrepared.EC$run(colMeans(samples.List.matrix.EC))

DIC.EC<--4*mean(logLiks.EC)+2*logLiks.meanparams.EC

p.DIC.EC<--2*mean(logLiks.EC)+2*logLiks.meanparams.EC

list(DIC.mode=DIC.mode.EC,p.DIC.mode=p.DIC.mode.EC,DIC=DIC.EC,p.DIC=p.DIC.EC)

}

)

out.mcmc<-runMCMC_btadjust(code=ModelCode, constants = ModelConsts, data = ModelData, MCMC_language="Nimble",

Nchains=Nchains, params=params, inits=Inits,

niter.min=1000, niter.max=300000,

nburnin.min=100, nburnin.max=200000,

thin.min=1, thin.max=1000,

conv.max=1.05, neff.min=1000,

control=list(print.diagnostics=FALSE),

control.MCMC=list(includeParentNodes=TRUE,extraCalculations=calculations.for.modalDIC))

attributes(out.mcmc)<-c(attributes(out.mcmc),DIC=NA)

The run ends up - in Linux but not in Windows- with the following messages that are unclear to me:

[1] "###################################################################################"

[1] "Performing extra calculations"

[1] "###################################################################################"

Compiling

[Note] This may take a minute.

[Note] Use 'showCompilerOutput = TRUE' to see C++ compilation details.

Warning: 8 objects were found from a DLL

Called from: cppProjects[[i]]$unloadSO(check = TRUE, force = FALSE)

Warning message:

In cppProjects[[i]]$unloadSO(check = TRUE, force = FALSE) :

A DLL to be unloaded has non-zero (population_dot_sd_L2_UID_5__1, population_dot_mean_L1_UID_4__1, Nimble__1, population_dot_variance_L3_UID_6__1, mass_L6_UID_9__1, lifted_d1_over_sqrt_oPprecision_cP_L6_UID_8__1, precision_L4_UID_7__1, numberedObjects) objects that need to be finalized first. It was not unloaded.

Browse[1]>

debug: if (!force) return(NULL)

Browse[2]> attributes(out.mcmc)<-c(attributes(out.mcmc),DIC=NA)

Error: object 'out.mcmc' not found

Browse[2]>

debug: return(NULL)

[1] "Performing extra calculations"

[1] "###################################################################################"

Compiling

[Note] This may take a minute.

[Note] Use 'showCompilerOutput = TRUE' to see C++ compilation details.

Warning: 8 objects were found from a DLL

Called from: cppProjects[[i]]$unloadSO(check = TRUE, force = FALSE)

Warning message:

In cppProjects[[i]]$unloadSO(check = TRUE, force = FALSE) :

A DLL to be unloaded has non-zero (population_dot_sd_L2_UID_5__1, population_dot_mean_L1_UID_4__1, Nimble__1, population_dot_variance_L3_UID_6__1, mass_L6_UID_9__1, lifted_d1_over_sqrt_oPprecision_cP_L6_UID_8__1, precision_L4_UID_7__1, numberedObjects) objects that need to be finalized first. It was not unloaded.

Browse[1]>

debug: if (!force) return(NULL)

Browse[2]> attributes(out.mcmc)<-c(attributes(out.mcmc),DIC=NA)

Error: object 'out.mcmc' not found

Browse[2]>

debug: return(NULL)

I have tried several ways of installing nimble - except from github which I am unsure I can install on this station - but the result was the same with all of them.

Thank you in advance for your help.

Sincerely,

Frédéric

Aug 6, 2024, 7:33:00 PMAug 6

to frederic....@gmail.com, nimble-users

Hi Frédéric,

I was able to reproduce this problem on one of our Ubuntu 22.04 machines.

It looks like this is happening because the package developer of `runMCMCbtadjust` chose to use our `clearCompiled`

function, which we document as being potentially dangerous. I'm not sure why the developer is using that.

One work-around would be for you to download the `runMCMCbtadjust` source package and modify `runMCMC_btadjust.r` so that

`clearCompiled` is not called. Then you can rebuild and reinstall and I suspect it will work for you.

-chris

--

You received this message because you are subscribed to the Google Groups "nimble-users" group.

To unsubscribe from this group and stop receiving emails from it, send an email to nimble-users...@googlegroups.com.

To view this discussion on the web visit https://groups.google.com/d/msgid/nimble-users/2daad335-b62b-41fe-8b8d-ad830d981d1dn%40googlegroups.com.

Aug 6, 2024, 8:21:37 PMAug 6

to frederic....@gmail.com, nimble-users

Hi Frédéric,

Perry and Daniel pointed out to me that you are the package developer. Whoops.

Can you say more about why you were using `clearCompiled`?

This might also be an issue best moved to an issue on the nimble GitHub repo.

-Chris

Aug 7, 2024, 12:15:09 PMAug 7

to nimble-users

I thank you Chris for the answers. Yes I am the developer of runMCMCbtadjust.

I used clearCompiled for the reasons given there: https://groups.google.com/g/nimble-users/c/-eoYs__eg0o: R crash due to "maximal number of DLLs reached" outside Windows.

Is there an alternative to clearCompiled for this problem? If not, should I also apply clearCompiled on the compiled nimbleFunctions?

Sincerely,

Frédéric

Aug 8, 2024, 9:19:21 AMAug 8

to nimble-users

In addition to my previous mail: when adding at the end of my calculations.for.modalDIC expression (see new expression below):

try(nimble::clearCompiled(ClogProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalc.EC),silent=TRUE)

try({rm(ClogProbCalcPrepared.EC)})

try({rm(logProbCalcPrepared.EC)})

try({rm(logProbCalc.EC)})

gc(verbose=FALSE,full=TRUE)

I get rid of most of the messages- I had previously and only retain one error that I still do not completely understand:

try(nimble::clearCompiled(logProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalc.EC),silent=TRUE)

try({rm(ClogProbCalcPrepared.EC)})

try({rm(logProbCalcPrepared.EC)})

try({rm(logProbCalc.EC)})

gc(verbose=FALSE,full=TRUE)

I get rid of most of the messages- I had previously and only retain one error that I still do not completely understand:

Error: cannot allocate vector of size 1247.7 Gb

Execution halted

Execution halted

Am I in thge good dircetion?

Sincerely,

Frédéric

#additions referred to above:

try(nimble::clearCompiled(ClogProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalc.EC),silent=TRUE)

try({rm(ClogProbCalcPrepared.EC)})

try({rm(logProbCalcPrepared.EC)})

try({rm(logProbCalc.EC)})

gc(verbose=FALSE,full=TRUE)

list(DIC.mode=DIC.mode.EC,p.DIC.mode=p.DIC.mode.EC,DIC=DIC.EC,p.DIC=p.DIC.EC)

}

)

try(nimble::clearCompiled(ClogProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalcPrepared.EC),silent=TRUE)

try(nimble::clearCompiled(logProbCalc.EC),silent=TRUE)

try({rm(ClogProbCalcPrepared.EC)})

try({rm(logProbCalcPrepared.EC)})

try({rm(logProbCalc.EC)})

gc(verbose=FALSE,full=TRUE)

list(DIC.mode=DIC.mode.EC,p.DIC.mode=p.DIC.mode.EC,DIC=DIC.EC,p.DIC=p.DIC.EC)

}

)

Aug 8, 2024, 11:29:32 AMAug 8

to frederic....@gmail.com, nimble-users

Hi Frédéric,

I haven't had time to try to dig into this in any substantial way. If you're able to avoid the errors, that seems good. In general one shouldn't need to run `gc()` as R will do that automatically on a regular basis, but perhaps it's needed here.

As far as the memory error, that's simply that R is running out of memory. In some cases, NIMBLE uses a lot of memory, unfortunately. I suspect if you ran the code on a smaller problem or used a machine with more memory that that error would go away.

-chris

To view this discussion on the web visit https://groups.google.com/d/msgid/nimble-users/e117c552-ea46-4419-b8db-ee99f16cb21fn%40googlegroups.com.

Reply all

Reply to author

Forward

0 new messages

Search

Clear search

Close search

Google apps

Main menu