[emt] r214 committed - This adds logging for plugins that time out. The current plugin execut...

[e...@googlecode.com]

Revision: 214
Author: eric.bergen
Date: Fri Oct 8 10:14:42 2010
Log: This adds logging for plugins that time out. The current plugin
execution method uses popen and I don't know of a good way to timeout the
fread calls other than potentially walking the process pree and killing
pids. The socket_read_timeout doesn't seem to work for popen'ed
descriptors. The proc_open status command required to get the pid of a
executed process is only available in php5. I will experiment further with
trying to get emt_gather to timeout bad plugins. This also adds a test
field and command which will time out. It's ok to see these errors during
the test run.

http://code.google.com/p/emt/source/detail?r=214

Modified:
/trunk/base/gather_general.php
/trunk/bin/emt_gather
/trunk/etc/emt.cnf
/trunk/plugins/commands/test.php
/trunk/plugins/fields/test.php
/trunk/test/etc/emt.cnf
/trunk/test/test.sh

=======================================
--- /trunk/base/gather_general.php Mon Dec 28 18:21:42 2009
+++ /trunk/base/gather_general.php Fri Oct 8 10:14:42 2010
@@ -68,9 +68,9 @@

function read_output()
{
- $safe_read = 15;
$block = '';

+
do {
$block = fread($this->fp, 1024);
$this->data .= $block;
=======================================
--- /trunk/bin/emt_gather Fri Jun 11 13:11:25 2010
+++ /trunk/bin/emt_gather Fri Oct 8 10:14:42 2010
@@ -33,6 +33,7 @@
* Write the aggregated, filtered, tested data to the output.

*/
+declare(ticks = 1);

require_once(dirname(__FILE__) . '/../base/config.php');
require_once(INSTALL_PATH . 'base/gather_general.php');
@@ -53,6 +54,27 @@

if (!count($config['emt_gather']['field']))
log_error_and_exit("No valid fields were found in the config file");
+
+if (isset($config['global']['plugin_timeout']))
+ $plugin_timeout = $config['global']['plugin_timeout'];
+else
+ $plugin_timeout = 10;
+
+$start_time = time();
+pcntl_signal(SIGALRM, "plugin_timeout_handler", true);
+$READ_TIMED_OUT = FALSE;
+
+function plugin_timeout_handler()
+{
+ global $READ_TIMED_OUT;
+ global $c;
+ global $plugin_timeout;
+
+ $READ_TIMED_OUT = TRUE;
+
+ log_error("Command {$c->cmd} from class " . get_class($c) . " timed
out.");
+ pcntl_alarm($plugin_timeout);
+}

$output_handlers = array();

@@ -109,6 +131,8 @@
$sem->lock();

$c_incr = -1;
+
+
foreach (array_keys($u_cmd) as $cmd)
{
if (isset($module_instances[$cmd]))
@@ -145,10 +169,24 @@
for ($i = 0; isset($output_handlers[$i]); $i++)
$output_handlers[$i]->open();

+
for ($i = 0; isset($commands[$i]); $i++)
{
$c =& $commands[$i];
+
+ if (time() - $start_time > $gather_time)
+ {
+ //print "time is $plugin_timeout\n";
+ pcntl_alarm($plugin_timeout);
+ }
+ else
+ {
+ //print "time is " . ($plugin_timeout + $gather_time) . "\n";
+ pcntl_alarm($plugin_timeout + $gather_time);
+ }
+
$ret = $c->read_output();
+ pcntL_alarm(0);

/**
* The command logged an error so we want to skip processing all the
fields
=======================================
--- /trunk/etc/emt.cnf Tue Jul 13 16:08:06 2010
+++ /trunk/etc/emt.cnf Fri Oct 8 10:14:42 2010
@@ -1,6 +1,7 @@
[global]
extra_config_dir=/etc/emt.d/
gather_time = 60
+plugin_timeout = 10

[emt_gather]
field=timestamp
=======================================
--- /trunk/plugins/commands/test.php Thu Jun 17 14:00:27 2010
+++ /trunk/plugins/commands/test.php Fri Oct 8 10:14:42 2010
@@ -38,3 +38,11 @@
",test_multiple.bar=" . (3 + $var);
}
}
+
+class time_me_out extends gather_general
+{
+ function command()
+ {
+ $this->cmd = "sleep " . ($this->gather_time + 10);
+ }
+}
=======================================
--- /trunk/plugins/fields/test.php Thu Feb 4 21:48:25 2010
+++ /trunk/plugins/fields/test.php Fri Oct 8 10:14:42 2010
@@ -44,3 +44,15 @@
var $multi_value = TRUE;
}

+register_field('time_me_out');
+class time_me_out_field extends csv_field
+{
+ var $namespace = 'test';
+ var $name = 'time_me_out';
+ var $fname = 'Time Me Out';
+ var $description = 'This field will take 10 seconds longer than
gather_time to complete';
+ var $type = 'avg';
+ var $precision = 0;
+ var $command = 'time_me_out';
+}
+
=======================================
--- /trunk/test/etc/emt.cnf Fri Jun 25 15:45:34 2010
+++ /trunk/test/etc/emt.cnf Fri Oct 8 10:14:42 2010
@@ -1,10 +1,11 @@
[global]
-gather_time = 60
+gather_time = 5

[emt_gather]
field=timestamp
field=test_single
field=test_multiple
+field=time_me_out
output_handler=local_text
instances = 2

=======================================
--- /trunk/test/test.sh Mon Aug 16 21:49:09 2010
+++ /trunk/test/test.sh Fri Oct 8 10:14:42 2010
@@ -2,6 +2,9 @@
SED_CMD="s/[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:){2}[0-9]{2}/2010-06-25
15:09:06/"
rm run/*

+echo "Test configuration"
+../bin/emt_dump_conf
+
echo "Testing gather"
../bin/emt_gather
sed -r "$SED_CMD" < run/gather > /tmp/t;