[roster-dns-management] r1149 committed - Change DnsCheckConfig to run named-checkconfig and named-checkzone in ...

[roster-dns...@googlecode.com]

Revision: 1149
Author: dcha...@gmail.com
Date: Fri Jul 12 12:52:52 2013
Log: Change DnsCheckConfig to run named-checkconfig and
named-checkzone in parallel

http://code.google.com/p/roster-dns-management/source/detail?r=1149

Modified:
/trunk/roster-config-manager/scripts/dnscheckconfig
/trunk/test/dnscheckconfig_test.py

=======================================
--- /trunk/roster-config-manager/scripts/dnscheckconfig Wed Feb 13 18:55:37
2013
+++ /trunk/roster-config-manager/scripts/dnscheckconfig Fri Jul 12 12:52:52
2013
@@ -46,6 +46,8 @@
from fabric import api as fabric_api
from fabric import state as fabric_state

+from multiprocessing import Pool
+
from optparse import OptionParser

from roster_config_manager import config_lib
@@ -89,6 +91,30 @@

finally:
file_handle.close()
+
+def RunFabricNamedCheckTool(args):
+ """Runs a fabric_api.local() call for named-check(conf|zone)
+
+ Inputs:
+ args: dictionary of args to be used for returning result
+ from a parallel fabric named-check(conf|zone) call. ex:
+ {'filename': 'root_config_dir/localhost/named/named.conf.a',
+ 'command': 'named-checkconf
root_config_dir/localhost/named/named.conf.a'}
+ -- OR --
+ {'filename': 'root_config_dir/localhost/public/sub.abc.lcl.db',
+ 'command':
+ 'named-checkzone
root_config_dir/localhost/public/sub.abc.lcl.db'}
+
+ Outputs:
+ dictionary of filename and named-check(conf|zone) output. ex:
+ {'filename': 'root_config_dir/university.edu/named.conf.a',
+ 'output': ''}
+ """
+ filename = args['filename']
+ command = args['command']
+ fabric_output = fabric_api.local(command, capture=True)
+ return {'filename': filename,
+ 'output': fabric_output}

def main(args):
"""Collects command line arguments. Checks configs.
@@ -145,17 +171,28 @@
fabric_state.output['running'] = False
fabric_state.output['warnings'] = False
fabric_api.env.warn_only = True
+
+ checkconf_pool = Pool(processes=config_lib_instance.max_threads)
+ parallel_checkconf_args_list = []
+
named_files = glob.glob('%s/*/named.conf.a' % (
config_lib_instance.root_config_dir))
for current_file in named_files:
ChangeNamedDirectory(current_file,
config_lib_instance.root_config_dir)
- checknamed_text = fabric_api.local('%s %s' % (
- options.named_checkconf, current_file), capture=True)
- if( checknamed_text == '' ):
+ command = '%s %s' % (options.named_checkconf, current_file)
+ parallel_checkconf_args_list.append({'filename': current_file,
+ 'command': command})
+
+ checkconf_results = checkconf_pool.map(RunFabricNamedCheckTool,
+ parallel_checkconf_args_list)
+
+ for result in checkconf_results:
+ named_file_output = result['output']
+ if( named_file_output == '' ):
if( options.verbose ):
- print 'Finished - %s' % current_file
+ print 'Finished - %s' % result['filename']
else:
- print 'ERROR: %s' % checknamed_text
+ print 'ERROR: %s' % named_file_output
sys.exit(1)

#Turns ['temp_dir/localhost/named.conf.a'] into ['temp_dir/localhost']
@@ -177,6 +214,9 @@
global_options_dict = named_file_dict['options']
zone_files = glob.glob('%s/*/*/*.db' % named_dir)

+ checkzone_pool = Pool(processes=config_lib_instance.max_threads)
+ parallel_checkzone_args_list = []
+
for current_file in zone_files:
#example - current_file =
#'root_config_dir/localhost/named/test_view/sub.university.lcl.db
@@ -201,7 +241,7 @@
break
else:
print('Zone file %s is in the named directory %s, '
- 'but not in it\'s named.conf' % (current_file, named_dir))
+ 'but not in it\'s named.conf' % (current_file, named_dir))
sys.exit(1)

file_handle = open(current_file, 'r')
@@ -219,18 +259,25 @@

additional_args_string = config_lib_instance.GetNamedZoneToolArgs(
current_server_name, view_name, current_zone_file)
- checkzone_command = ('%s %s %s %s' % (
- options.named_checkzone, additional_args_string,
- origin, current_file))
+
+ #Run the named_checkzone command concurrently
+ command = '%s %s %s %s' % (
+ options.named_checkzone, additional_args_string, origin,
current_file)
+
+ parallel_checkzone_args_list.append({'filename': current_file,
+ 'command': command})

- checkzone_text = fabric_api.local(checkzone_command, capture=True)
+ results = checkzone_pool.map(RunFabricNamedCheckTool,
+ parallel_checkzone_args_list)

- if( checkzone_text.endswith('\nOK') ):
- if( options.verbose ):
- print 'Finished - %s' % current_file
- else:
- print 'ERROR: %s' % checkzone_text
- sys.exit(1)
+ for result in results:
+ checkzone_output = result['output']
+ if( checkzone_output.endswith('\nOK') ):
+ if( options.verbose ):
+ print 'Finished - %s' % result['filename']
+ else:
+ print 'ERROR: %s' % checkzone_output
+ sys.exit(1)

if( options.verbose ):

print '--------------------------------------------------------------------'
=======================================
--- /trunk/test/dnscheckconfig_test.py Fri May 10 09:51:01 2013
+++ /trunk/test/dnscheckconfig_test.py Fri Jul 12 12:52:52 2013
@@ -240,6 +240,105 @@
self.assertEqual(output.read(), '')
output.close()

+ def testCheckConfigParallelSpeedup(self):
+ self.assertEqual(self.core_instance.ListRecords(), [])
+ self.core_instance.MakeACL(u'internal', u'127.0.0.1')
+ self.core_instance.MakeView(u'external')
+ self.core_instance.MakeDnsServer(u'localhost', SSH_USER, BIND_DIR,
TEST_DIR)
+ self.core_instance.MakeDnsServerSet(u'set1')
+ self.core_instance.MakeDnsServerSetAssignments(u'localhost', u'set1')
+ self.core_instance.MakeDnsServerSetViewAssignments(u'external', 1,
u'set1')
+ self.core_instance.MakeViewToACLAssignments(u'external', u'set1',
+ u'internal', 1)
+
+ #Global options
+ self.core_instance.MakeNamedConfGlobalOption(u'set1',
+ u'options { check-names master ignore; };\n')
+
+ #Test single zone check
+ self.core_instance.MakeZone(u'test_zone0', u'master',
+ u'test_zone0.university.lcl.',
view_name=u'external')
+ self.core_instance.MakeRecord(
+ u'soa', u'@', u'test_zone0',
+ {u'name_server': u'ns.university.lcl.',
+ u'admin_email': u'admin.university.edu.',
+ u'serial_number': 1, u'refresh_seconds': 5,
+ u'retry_seconds': 5, u'expiry_seconds': 5,
+ u'minimum_seconds': 5}, view_name=u'external')
+
+ self.core_instance.MakeRecord(
+ u'ns', u'@', u'test_zone0',
+ {u'name_server': u'ns.university.lcl.'},
+ view_name=u'external')
+
+ for j in range(20):
+ self.core_instance.MakeRecord(u'a', u'machine%s' % j,
+ u'test_zone0',
+ {u'assignment_ip': u'10.10.10.0'},
+ view_name=u'external')
+
+ self.tree_exporter_instance.ExportAllBindTrees()
+ time.sleep(2) # Wait for disk to settle
+
+ #audit_log_id, filename =
self.config_lib_instance.FindNewestDnsTreeFilename()
+ #self.config_lib_instance.UnTarDnsTree(audit_log_id)
+
+ single_start_time = time.time()
+ command = subprocess.Popen('python %s --config-file %s --verbose' % (
+ EXEC, CONFIG_FILE), shell=True, stdout=subprocess.PIPE)
+ #Wait for execution to finish
+ output = command.communicate()[0]
+
+ single_elapsed_time = time.time() - single_start_time
+
+ output_lines = output.split('\n')
+ self.assertTrue('Checked 1 named.conf file(s) and 1 zone file(s)' in
+ output_lines)
+ self.assertTrue('All checks successful' in output_lines)
+
+ #Test multiple zones and measure speedup
+ #Create enough zones/records to see time difference in parallel
execution
+ for i in range(1, 50):
+ self.core_instance.MakeZone(u'test_zone%s' % i, u'master',
+ u'test_zone%s.university.lcl.' % i,
+ view_name=u'external')
+ self.core_instance.MakeRecord(
+ u'soa', u'@', u'test_zone%s' % i,
+ {u'name_server': u'ns.university.edu.',
+ u'admin_email': u'admin.university.edu.',
+ u'serial_number': 1, u'refresh_seconds': 5,
+ u'retry_seconds': 5, u'expiry_seconds': 5,
+ u'minimum_seconds': 5}, view_name=u'external')
+
+ self.core_instance.MakeRecord(u'ns', u'@', u'test_zone%s' % i,
+ {u'name_server':
u'ns.university.lcl.'},
+ view_name=u'external')
+
+ for j in range(20):
+ self.core_instance.MakeRecord(u'a', u'machine%s' % j,
+ u'test_zone%s' % i,
+ {u'assignment_ip': u'10.10.10.0'},
+ view_name=u'external')
+
+ self.tree_exporter_instance.ExportAllBindTrees()
+ time.sleep(2) # Wait for disk to settle
+
+ multi_start_time = time.time()
+ command = subprocess.Popen('python %s --config-file %s --verbose' % (
+ EXEC, CONFIG_FILE), shell=True, stdout=subprocess.PIPE)
+ #Wait for execution to finish
+ output = command.communicate()[0]
+ multi_elapsed_time = time.time() - multi_start_time
+
+ output_lines = output.split('\n')
+
+ # Assert speedup
+ self.assertTrue(multi_elapsed_time < single_elapsed_time * 25)
+
+ self.assertTrue('Checked 1 named.conf file(s) and 50 zone file(s)' in
+ output_lines)
+ self.assertTrue('All checks successful' in output_lines)
+
def testCheckServerInfo(self):
self.assertEqual(self.core_instance.ListRecords(), [])
output = os.popen('python %s -f test_data/test_zone.db '