Batch List - Batch Stitcher Multi-Thread

[Ro M]

Hi, I have around 10,000 projects that need to be processed, but the processing is sequential, one by one. My system isn't fully utilized, with only about 30% CPU and 3% GPU usage. Is there any way to speed up this process?

[Erik Krause]

Am 03.10.2024 um 20:14 schrieb Ro M:

> My system isn't fully utilized, with
> only about 30% CPU and 3% GPU usage.

Perhaps https://ptgui.com/support.html#3_15 has a clue.

--
Erik Krause
http://www.erik-krause.de

[PTGui Support]

The slowness might actually be caused by saving the updated batch list
after each stitch. If you have a Batch List loaded, the batch stitcher
will record the state of each project in the batch list (Already
processed vs not yet processed). After stitching each project, the batch
list is updated on disk. If you have small projects and a large batch
list, this updating takes relatively much time compared to the stitching
time.

It can therefore be helpful to uncheck 'File - Automatically Save Batch
List'.

PTGui currently doesn't allow running multiple batch stitchers in
parallel, although there's a hack you might try: if you install multiple
PTGui versions side by side, each one can run a batch stitcher. So you
can download another PTGui version (PTGui 12.26 or 13 beta 7) and run it
at the same time.

See 1.8:
https://ptgui.com/support.html#1_8

and 3.29:
https://ptgui.com/support.html#3_29

Kind regards,

Joost Nieuwenhuijse
www.ptgui.com

[Ro M]

Thanks for the responses!

Found the solution.

Currently, I have version 13 beta 8, and I use  Node.js  script to run N instances of stitching.

In my case, 4 instances are enough to load my CPU and GPU to a 80-100%. 

For 2000 projects from the list, the process takes about 15 minutes, and I am totally satisfied with the results.

My specs: 13600KF, 4070 Ti, 64 GB RAM at 6 GHz,  4 TB NVMe at 7 GB/s.

If anyone has similar needs, you can use this Node.js script. (node v18 + xml2js module)

Just update the file name, which in my case is "Batch List.ptgbatch":

const batchListFile = path.join(__dirname, 'Batch List.ptgbatch');

Path to PTGui (I have PTGui globally defined in Windows Path variables):

const ptguiExe = 'PTGui.exe';

And the maximum number of concurrent processes:

const maxConcurrentProcesses = 4;

to use the script:

1. Save the code to a file named script.js.
2. Run it with Node.js using the command: node script.js.

const fs = require('fs');
const path = require('path');
const { exec } = require('child_process');
const xml2js = require('xml2js');

// Path to the PTGui Batch List file
const batchListFile = path.join(__dirname, 'Batch List.ptgbatch');

// PTGui executable path
const ptguiExe = 'PTGui.exe';

// Maximum number of concurrent processes
const maxConcurrentProcesses = 4;

// Variables to track progress and time
let totalProjects = 0;
let completedProjects = 0;
let startTime = 0;

// Function to format time (seconds) into HH:MM:SS
function formatTime(seconds) {
  const hrs = Math.floor(seconds / 3600);
  const mins = Math.floor((seconds % 3600) / 60);
  const secs = Math.floor(seconds % 60);
  return `${hrs.toString().padStart(2, '0')}:${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
}

// Function to display current progress and ETA
function displayProgress() {
  completedProjects++;
  const elapsedTime = (Date.now() - startTime) / 1000; // in seconds
  const avgTimePerProject = elapsedTime / completedProjects;
  const remainingProjects = totalProjects - completedProjects;
  const eta = avgTimePerProject * remainingProjects;

  console.log(`Progress: ${completedProjects} of ${totalProjects} completed.`);
  console.log(`ETA: ${formatTime(eta)} remaining.`);
}

// Function to stitch a .pts file
function stitchProject(projectPath, callback) {
  console.log(`Starting stitching for: ${projectPath}`);
 
  // Update the command to use -stitchnogui
  const cmd = `"${ptguiExe}" -stitchnogui "${projectPath}"`;

  exec(cmd, (error, stdout, stderr) => {
    if (error) {
      console.error(`Error stitching ${projectPath}:`, error);
    } else {
      console.log(`Completed stitching for: ${projectPath}`);
      console.log(stdout);  // Log stitching progress
      displayProgress();
    }
    callback();
  });
}

// Function to process the .ptgbatch file
function processBatchFile() {
  // Read the batch list XML file
  fs.readFile(batchListFile, (err, data) => {
    if (err) {
      console.error('Error reading Batch List.ptgbatch:', err);
      return;
    }

    // Parse the XML file
    xml2js.parseString(data, (parseErr, result) => {
      if (parseErr) {
        console.error('Error parsing XML:', parseErr);
        return;
      }

      const projects = result.PTGuiBatchList.Project
        .map((proj) => proj.$.FileName)
        .filter(Boolean);

      totalProjects = projects.length; // Set total projects
      startTime = Date.now(); // Record the start time

      // Process projects concurrently with a maximum of 8 processes at a time
      runConcurrentProcesses(projects, maxConcurrentProcesses);
    });
  });
}

// Function to run concurrent stitching processes
function runConcurrentProcesses(projects, maxConcurrent) {
  let runningProcesses = 0;
  let currentIndex = 0;

  function next() {
    if (currentIndex >= projects.length) {
      // All projects processed
      if (runningProcesses === 0) {
        console.log('All stitching tasks completed.');
      }
      return;
    }

    // If less than the maxConcurrent processes are running, start a new one
    while (runningProcesses < maxConcurrent && currentIndex < projects.length) {
      const projectPath = projects[currentIndex];
      runningProcesses++;
      currentIndex++;

      stitchProject(projectPath, () => {
        runningProcesses--;
        next();
      });
    }
  }

  // Start the stitching process
  next();
}

// Start processing the batch file
processBatchFile();