Multithreading PyEZ Devices

[Ansel Gaddy]

Please show me how to modify this for multithreading. I use the try and except to bypass any routers who do not accept my standard login credentials there may be a better way to do that.

Normal Operation (takes a long time)

user@host:~/dev/scripts/staticCleanup$ python2.7

Python 2.7.3 (default, May 23 2013, 09:27:41)

[GCC 4.1.2 20080704 (Red Hat 4.1.2-52)] on linux2

Type "help", "copyright", "credits" or "license" for more information.

>>> from getRouters import getRouters

>>> z = getRouters('username','password')

Got 306 IPs

Can reach <ip1> resolving hostname

Can reach <ip2> resolving hostname

Can reach <ip3> resolving hostname

Can reach <ip4> resolving hostname

Can reach <ip5> resolving hostname

Cant reach <ip6>

Can reach <ip7> resolving hostname

Can reach <ip8> resolving hostname

Can reach <ip9> resolving hostname

etc..

Result is a list of dictionaries with keys 'ip' and 'name' , unreachable routers are removed from the list of dictionaries on purpose

>>> len(z)

280

>>> z[0]['ip']

'xx.xx.xx.xx'

>>> z[0]['name']

'xxxxxxxx1cw'

### begin python code

from jnpr.junos import Device

from pprint import pprint

import getpass

import xmldict

# gets routers from H route reflector

def getRouters(u,p):

dev = Device(host='<route reflector H ip>', user=u, password=p)

dev.open(gather_facts=False)

configXML = dev.rpc.get_configuration()

dev.close()

configDic = xmldict.xml_to_dict(configXML)

ips = []

for i in configDic['configuration']['protocols']['bgp']['group']:

if i['name']=='CEN-IBGP-RS-CLIENTS':

for z in i['neighbor']:

ips.append( z['name'] )

print 'Got',len(ips),'IPs'

neighbors=[]

#this is the part that I am trying to multithread

for i in ips:

dev = Device(host=i, user=u, password=p)

if dev.probe():

print 'Can reach',i,'resolving hostname'

try:

dev.open(gather_facts=False)

configXML = dev.rpc.get_configuration()

dev.close()

hname = configXML.xpath('.//system/host-name')[0].text

hname = hname.split('.')[1]

neighbors.append({'ip':i,'name':hname})

except:

ips.remove(i)

else:

print 'Cant reach',i

ips.remove(i)

return neighbors

if __name__=='__main__':

u = raw_input("User: ")

p = getpass.getpass()

pprint( getRouters(u,p) )

### end python code

Thanks again for all your help and your awesome product!

Sincerely,

-Ansel

[Rick Sherman]

Hi Ansel,

Here is some code i put together that *may* work for you.  It's untested and a little dirty using global variables, but should get you started.

The threads right now are set to double the cores - that should be able to scale higher.  

There is also a way to filter the configuration request - I don't have time tonight to update that portion, but it would look something like:

# Config filtering
from lxml.builder import E
get = E('configuration', E('protocols', E('bgp', E('group'))))
configXML = dev.rpc.get_config(get)

Here is the untested threaded version:

from jnpr.junos import Device

from pprint import pprint

import getpass

import xmldict

from lxml.builder import E

from multiprocessing.dummy import Pool as ThreadPool

from multiprocessing import cpu_count

neighbors = []

u = ""

p = ""

# gets routers from H route reflector

def getRouters(u, p):

    dev = Device(host='<route reflector H ip>', user=u, password=p)

    dev.open(gather_facts=False)

    configXML = dev.rpc.get_configuration()

    dev.close()

    configDic = xmldict.xml_to_dict(configXML)

    ips = []

    for i in configDic['configuration']['protocols']['bgp']['group']:

        if i['name'] == 'CEN-IBGP-RS-CLIENTS':

            for z in i['neighbor']:

                ips.append(z['name'])

    print 'Got', len(ips), 'IPs'

    return ips

def getHosts(i):

    global u

    global p

    global neighbors

    dev = Device(host=i, user=u, password=p)

    if dev.probe():

        print 'Can reach', i, 'resolving hostname'

        try:

            dev.open(gather_facts=False)

            configXML = dev.rpc.get_configuration()

            dev.close()

            hname = configXML.xpath('.//system/host-name')[0].text

            hname = hname.split('.')[1]

            neighbors.append({'ip': i, 'name': hname})

        except:

            pass

    else:

        print 'Cant reach', i

    return

if __name__ == '__main__':

    global u

    global p

    global neighbors

    u = raw_input("User: ")

    p = getpass.getpass()

    ips = getRouters(u, p)

    # 2 threads per core - can probably set this upwards of 16

    pool = ThreadPool(cpu_count() * 2)

    pool.map_async(getHosts, ips)

    pool.close()

    pool.join()

    pprint(neighbors)

Let me know if this works.

-Rick

[Ansel Gaddy]

I initially tried something very close to this based on the multithreading example you had sent. I am getting an identical result with your code. Neighbors is not being populated with dictionaries.

In my previous attempt I changed getHosts() to return {'ip':i,'name':hname} that way I wouldnt need global variables but this had the exact same result as your code. This leads me to think its something with the way map_async works.

user@host:~/dev/scripts/staticCleanup$ python2.7 -i multiGetRouters.py

multiGetRouters.py:52: SyntaxWarning: name 'u' is assigned to before global declaration

  global u

multiGetRouters.py:53: SyntaxWarning: name 'p' is assigned to before global declaration

  global p

multiGetRouters.py:54: SyntaxWarning: name 'neighbors' is assigned to before global declaration

  global neighbors

User: *****

Password:

Got 306 IPs

Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Can ping Cant reach[]

--- 6.01802110672 seconds ---

>>> dir()

['Device', 'ThreadPool', '__builtins__', '__doc__', '__name__', '__package__', 'cpu_count', 'getHosts', 'getRouters', 'getpass', 'ips', 'neighbors', 'p', 'pool', 'pprint', 'start_time', 'time', 'u', 'xmldict']

>>> neighbors

[]

>>> type(neighbors)

<type 'list'>

>>> len(neighbors)

0

>>> dir()

['Device', 'ThreadPool', '__builtins__', '__doc__', '__name__', '__package__', 'cpu_count', 'getHosts', 'getRouters', 'getpass', 'ips', 'neighbors', 'p', 'pool', 'pprint', 'start_time', 'time', 'u', 'xmldict']

This is not critical to complete this evening so no hurry, but I am looking forward to the solution to this interesting problem.

Thanks again!

Sincerely,

-Ansel

[Rick Sherman]

Hi Ansel,

Here is a cleaned up and tested version of the script:

#!/usr/bin/python

import sys

from pprint import pprint
import getpass

from lxml.builder import E
from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import cpu_count

from jnpr.junos import Device
from jnpr.junos.exception import ConnectError, RpcError


class test():

    def __init__(self):
        self.u = raw_input("User: ")
        self.p = getpass.getpass()

    # gets routers from H route reflector

    def getRouters(self):
        dev = Device(host='192.168.74.31', user=self.u, password=self.p)
        print "Connecting to route reflector"

        try:
            dev.open(gather_facts=False)

            get = E('configuration', E('protocols', E('bgp', E('group', E('name', 'CEN-IBGP-RS-CLIENTS')))))
            conf = dev.rpc.get_config(get)
            dev.close()

        except ConnectError as err:
            print "Unable to connect to route reflector!"
            print err
            sys.exit()

        except RpcError as err:
            print "Unable to get neighbors!"
            print err
            dev.close()
            sys.exit()

        neighbors = conf.findall('.//neighbor/name')

        ips = []
        for i in neighbors:
            ips.append(i.text)

        print 'Got {0} IPs'.format(len(ips))
        return ips

    def getHosts(self, i):
        dev = Device(host=i, user=self.u, password=self.p)
        if dev.probe():
            print 'Can reach {0} - resolving hostname'.format(i)
            try:
                dev.open(gather_facts=False)
                get = E('configuration', E('system', E('host-name')))
                configXML = dev.rpc.get_config(get)
                dev.close()
                hname = configXML.findtext('.//host-name')

                hname = hname.split('.')[1]

                return {'ip': i, 'name': hname}

            except:
                pass
        else:
            print 'Cant reach {0}'.format(i)

        return


if __name__ == '__main__':

    main = test()
    ips = main.getRouters()

    # 2 threads per core - can probably set this upwards of 16
    pool = ThreadPool(cpu_count() * 2)

    results = pool.map_async(main.getHosts, ips)
    pool.close()
    pool.join()
    neighbors = filter(None, results.get())
    pprint(neighbors)

Please give this a shot and let me know how it works out for you.

Thanks,

-Rick

[Ansel Gaddy]

Hi Rick,

The multithreading part works beautifully, but I am getting an empty neighbors variable at the end.

I tried renaming neighbors to nei in __main__ in case it was conflicting with the other variable named neighbors but this did not fix it.

<user>@<host>:~/dev/scripts/staticCleanup$ python2.7 -i multiGetRouters.py

User: <user>

Password:

Connecting to route reflector

Got 306 IPs

Can reach <ip1> - resolving hostname

etc..

Can reach <ip306>- resolving hostname

[]

>>> dir()

['ConnectError', 'Device', 'E', 'RpcError', 'ThreadPool', '__builtins__', '__doc__', '__name__', '__package__', 'cpu_count', 'getpass', 'ips', 'main', 'nei', 'pool', 'pprint', 'results', 'sys', 'test']

>>> len(ips)

306

>>> len(nei)

0

if __name__ == '__main__':

    main = test()

    ips = main.getRouters()

    # 2 threads per core - can probably set this upwards of 16

    pool = ThreadPool(cpu_count() * 4)

    results = pool.map_async(main.getHosts, ips)

    pool.close()

    pool.join()

    nei = filter(None, results.get())

    pprint(nei)

Sincerely,

-Ansel

[Ansel Gaddy]

The issue was processing the XML to get the hostname. I added the XML config from each router to the results because I am using this anyways. This works great!

Example results

 {'ip': '<ip1>',

  'name': '<hostname1>',

  'xml': <Element configuration at 0xb126d40>},

 {'ip': '<ip2>',

  'name': '< hostname2>',

  'xml': <Element configuration at 0xb147950>}

Here is the working version for those interested.

#!/usr/bin/python

import sys

from pprint import pprint

import getpass

from lxml.builder import E

from multiprocessing.dummy import Pool as ThreadPool

from multiprocessing import cpu_count

from jnpr.junos import Device

from jnpr.junos.exception import ConnectError, RpcError

class multiGetRouters():

    def __init__(self,u,p):

self.usr = u

self.pw = p

self.ips = None

self.results = []

self.getRouters()

self.multiGetHosts()

    # gets routers from H route reflector

    def getRouters(self):

        dev = Device(host='<Route Reflector H IP>', user=self.usr, password=self.pw)

        print "Connecting to route reflector"

        try:

            dev.open(gather_facts=False)

            get = E('configuration', E('protocols', E('bgp', E('group', E('name', 'CEN-IBGP-RS-CLIENTS')))))

            conf = dev.rpc.get_config(get)

            dev.close()

        except ConnectError as err:

            print "Unable to connect to route reflector!"

            print err

            sys.exit()

        except RpcError as err:

            print "Unable to get neighbors!"

            print err

            dev.close()

            sys.exit()

        neighbors = conf.findall('.//neighbor/name')

        ips = []

        for i in neighbors:

            ips.append(i.text)

        print 'Got {0} IPs'.format(len(ips))

        self.ips = ips

    def getHosts(self, i):

        dev = Device(host=i, user=self.usr, password=self.pw)

        if dev.probe():

            print 'Can reach {0} - resolving hostname'.format(i)

            try:

                dev.open(gather_facts=False)

                configXML = dev.rpc.get_configuration()

                dev.close()

                hname = configXML.xpath('.//system/host-name')[0].text

                hname = hname.split('.')[1]

                self.results.append( {'ip': i, 'name': hname, 'xml':configXML} )

            except:

                print 'Couldnt connect to',i

    

    def multiGetHosts(self):

    # 6 threads per core - dont set more than 6 for ~50% CPU

pool = ThreadPool(cpu_count() * 6)

results = pool.map_async(self.getHosts, self.ips)

pool.close()

pool.join()

if __name__ == '__main__':

u = raw_input("User: ")

p = getpass.getpass()

main = multiGetRouters(u,p)

pprint(main.results)

...