Parallel table random accuracy( help please!)

[ema...@umbc.edu]

Hi,

I am implementing a parallel model which has 4 parallel nets and then the output is concatenated and send to a classifier. I checked my label and inputs to the model and they are correct but the model give me random accuracy for a very simple task like giving two same image and the other two is the same order with label 1 and the opposite order with label 2.I was wondering if someone can kindly help me where the problem of my code is. this is the code that I wrote:

net =  nn.Sequential()
net1 =  nn.Sequential()

net1:add(nn.SpatialConvolution(3,96,11,11,4,4,2,2))      
net1:add(nn.ReLU(true))
net1:add(nn.SpatialMaxPooling(3,3,2,2))                  
net1:add(nn.SpatialConvolution(96,256,5,5,1,1,2,2))    
net1:add(nn.ReLU(true))
net1:add(nn.SpatialMaxPooling(3,3,2,2))                
net1:add(nn.SpatialConvolution(256,384,3,3,1,1,1,1))    
net1:add(nn.ReLU(true))
net1:add(nn.SpatialConvolution(384,256,3,3,1,1,1,1))    
net1:add(nn.ReLU(true))
net1:add(nn.SpatialConvolution(256,256,3,3,1,1,1,1))      -
net1:add(nn.SpatialBatchNormalization(256))
net1:add(nn.ReLU(true))
net1:add(nn.SpatialMaxPooling(3,3,2,2))            
net1:add(nn.View(-1):setNumInputDims(3))
net1:add(nn.Linear(256*6*6, 512))
net1:add(nn.Linear(224*224*3, 100))

parallel_model = nn.Parallel(2,2)
parallel_model:add(net1)
for i =1,3 do
       parallel_model:add(net1)
end

  net:add(parallel_model)
  net:add(nn.Linear(512*4, 4096))
  net:add(nn.ReLU())
  net:add(nn.Dropout(0.5))
  net:add(nn.Linear(4096, 4096))
  net:add(nn.ReLU())
  net:add(nn.Dropout(0.5))
  net:add(nn.Linear(4096, opt.nClasses))

local input1 = torch.Tensor(opt.batchSize,4, 3, opt.fineSize, opt.fineSize)
if opt.gpu > 0 then
 

  input1 = input1:cuda()
  label = label:cuda()
  net:cuda()
  net1:cuda()
  parallel_model:cuda()
  criterion:cuda()
 
end

if opt.gpu > 0 and opt.cudnn > 0 then
  require 'cudnn'
 
  net = cudnn.convert(net, cudnn)
  net1 = cudnn.convert(net1, cudnn)
  parallel_model = cudnn.convert(parallel_model, cudnn)
 
end

 output = net:forward(input1)
  err = criterion:forward(output, label)
  local df_do = criterion:backward(output, label)
  net:backward(input1, df_do)