I wrote a perl script that reads its input, and uses it as commands to
execute in parallel. It tries to guess the maximum number of parallel
jobs to run at any time, or accepts a --parallel argument telling it
how many jobs is the maximu,.
#!/usr/bin/perl
#
# This script reads its standard input, or from files given on
# command line (<>).
#
# It executes every line as a separate shell command. If --parallel
# argument is given, it executes as many jobs in parallel as
# possible, but no more tham "--parallel" at any given time.
#
# This can be helpful to speed up some tasks.
#
# The default level of parallel is based on the number of CPUs.
#
# You need module Parallel::ForkManager. It is available as
# an ubuntu package.
#
# Copyright(C) Igor Chudov, 2009. All rights reserved.
# This script is made available to the public under the latest
# GPL Version found at
http://www.gnu.org/licenses/gpl.html
#
# No warranty is given or implied. Refunds will not be provided.
#
# Igor Chudov,
http://igor.chudov.com/
#
use strict;
use warnings;
use Getopt::Long;
use Parallel::ForkManager;
my $parallel = undef;
GetOptions(
"parallel=i" => \$parallel,
);
unless( defined $parallel ) {
if( open( CPU, "/proc/cpuinfo" ) ) {
$parallel = 0;
while( <CPU> ) {
$parallel++ if /^processor\b/;
}
close( CPU );
} else {
$parallel = 0;
}
}
my $pm = new Parallel::ForkManager( $parallel );
while( <> ) {
chomp;
my $pid = $pm->start and next;
system( $_ );
$pm->finish;
}
$pm->wait_all_children;