Thread getting stuck\hang

[Iranna Mathapati]

Hi Team,

I have python code that creates thread.Sometimes it works, sometimes it
gets stuck once its finish its execution part. still not yet cannot find
any difference between when it works or when it gets stuck. Please help.


sniffer1 =
threading.Thread(target=validate_traffic_stats_dy,args=(FT_item_dy,RT_item_dy,forward_path_list_dy,return_path_list_dy,nat_type_list_dy,pkt_dy_1,))

sniffer2 =
threading.Thread(target=validate_traffic_stats_st,args=(FT_item_st,RT_item_st,forward_path_list_st,return_path_list_st,nat_type_list_st,pkt_st,))

sniffer1.start()

sniffer2.start()

sniffer3 = threading.Thread(target=delete_new_dynamic_nat_conf)

sniffer3.start()



sniffer1.join()

sniffer2.join()

sniffer3.join()



Thanks,

Iranna m

[Iranna Mathapati]

Hi Dennis,

all function arguments declare as global and pass it to the function.

All three thread is completed the execution part and after completion of
execution its stuck/hang the programme.

*def
validate_traffic_stats_st(FT_item,RT_item,forward_path_list,return_path_list,nat_type_list,pkt_st)*
:
global flag1
flag1 = 1

log.info('Starting Static NAT forward Traffic Test...')

returndict = itgen_explorer.startTrafficFlows(testbeddict,
log,'-traffic_items {0} -pktcount {1} -profile
{2}'.format(FT_item,pkt_st,itgenProfile))
time.sleep(5)
global_stats_list = []
temp = 0
forward_ip_list = []
return_ip_list = []
global count_st_Rx,count_st_in_Rx,count_st_out_Rx
count_st_Rx = 0
count_st_in_Rx = 0
count_st_out_Rx = 0
for data in returndict['txrxstats']['rx'].keys():
if "TI01" in data:
count_st_Rx += int(returndict['txrxstats']['rx'][data] )
count_st_in_Rx += int(returndict['txrxstats']['rx'][data] )
elif "TI22" in data:
count_st_Rx += int(returndict['txrxstats']['rx'][data] )
count_st_out_Rx += int(returndict['txrxstats']['rx'][data] )

global count_st_Tx,count_st_in_Tx,count_st_out_Tx
count_st_Tx = 0
count_st_in_Tx = 0
count_st_out_Tx = 0

for data in returndict['txrxstats']['tx'].keys():
if "TI01" in data:
count_st_Tx += int(returndict['txrxstats']['tx'][data] )
count_st_in_Tx += int(returndict['txrxstats']['tx'][data] )
elif "TI22" in data:
count_st_Tx += int(returndict['txrxstats']['tx'][data] )
count_st_out_Tx += int(returndict['txrxstats']['tx'][data] )


for forward_ip_list,return_ip_list,nat_type in zip
(forward_path_list,return_path_list,nat_type_list):
global_stats_list = []
for translation_ip,return_ip in zip
(forward_ip_list,return_ip_list):
output=listtostr(returndict['txrxstats']['rx'])
if re.search('{0}'.format(translation_ip),output,re.I):
pass
else:
global_stats_list.append(translation_ip)
if global_stats_list == []:
log.info('itgen traffic validation for {0} forward path -
verified'.format(nat_type))
forward_ip_list = []
return_ip_list = []
else:
log.error('{0} forward path itgen verification
failed'.format(nat_type))
flag1 = 0
return flag1
log.info("Trafiic validation for NAT return path starting")
returndict = itgen_explorer.startTrafficFlows(testbeddict,
log,'-traffic_items {0} -pktcount 1000 -profile
{1}'.format(RT_item,itgenProfile))
time.sleep(5)
local_stats_list = []
for forward_ip_list,return_ip_list,nat_type in zip
(forward_path_list,return_path_list,nat_type_list):
local_stats_list = []
for translation_ip,return_ip in zip
(forward_ip_list,return_ip_list):
output=listtostr(returndict['txrxstats']['rx'])
if re.search('{0}'.format(return_ip),output,re.I):
pass
else:
local_stats_list.append(return_ip)
if local_stats_list == []:
log.info('itgen traffic validation for {0} return path -
verified'.format(nat_type))
forward_ip_list = []
return_ip_list = []
else:
log.error('{0} return path itgen verification
failed'.format(nat_type))
flag1 = 0
return flag1

return
flag1,count_st_Rx,count_st_in_Rx,count_st_out_Rx,count_st_Tx,count_st_in_Tx,count_st_out_Tx

*def
validate_traffic_stats_dy(FT_item,RT_item,forward_path_list,return_path_list,nat_type_list,pkt_dy_1):*
global flag2
flag2 = 1
log.info('Starting Dynamic NAT forward Traffic Test...')

returndict = itgen_explorer.startTrafficFlows(testbeddict,
log,'-traffic_items {0} -pktcount {1} -profile
{2}'.format(FT_item,pkt_dy_1,itgenProfile))
time.sleep(5)
global_stats_list = []
forward_ip_list = []
return_ip_list = []
global count_dy_Rx,count_dy_in_Rx,count_dy_out_Rx
count_dy_Rx = 0
count_dy_in_Rx = 0
count_dy_out_Rx = 0
for data in returndict['txrxstats']['rx'].keys():
if "TI05" in data:
count_dy_Rx += int(returndict['txrxstats']['rx'][data] )
count_dy_in_Rx += int(returndict['txrxstats']['rx'][data] )
elif "TI66" in data:
count_dy_Rx += int(returndict['txrxstats']['rx'][data] )
count_dy_out_Rx += int(returndict['txrxstats']['rx'][data] )

global count_dy_Tx,count_dy_in_Tx,count_dy_out_Tx
count_dy_Tx = 0
count_dy_in_Tx = 0
count_dy_out_Tx = 0
for data in returndict['txrxstats']['tx'].keys():
if "TI05" in data:
count_dy_Tx += int(returndict['txrxstats']['tx'][data] )
count_dy_in_Tx += int(returndict['txrxstats']['tx'][data] )
elif "TI66" in data:
count_dy_Tx += int(returndict['txrxstats']['tx'][data] )
count_dy_out_Tx += int(returndict['txrxstats']['tx'][data] )

for forward_ip_list,return_ip_list,nat_type in zip
(forward_path_list,return_path_list,nat_type_list):
global_stats_list = []
for translation_ip,return_ip in zip
(forward_ip_list,return_ip_list):
output=listtostr(returndict['txrxstats']['rx'])
if re.search('{0}'.format(translation_ip),output,re.I):
pass
else:
global_stats_list.append(translation_ip)
if global_stats_list == []:
log.info('itgen traffic validation for {0} forward path -
verified'.format(nat_type))
forward_ip_list = []
return_ip_list = []
else:
log.error('{0} forward path itgen verification
failed'.format(nat_type))
flag2 = 0
return flag2
log.info("Trafiic validation for NAT return path starting")
returndict = itgen_explorer.startTrafficFlows(testbeddict,
log,'-traffic_items {0} -pktcount 1000 -profile
{1}'.format(RT_item,itgenProfile))
time.sleep(5)
local_stats_list = []
for forward_ip_list,return_ip_list,nat_type in zip
(forward_path_list,return_path_list,nat_type_list):
local_stats_list = []
for translation_ip,return_ip in zip
(forward_ip_list,return_ip_list):
output=listtostr(returndict['txrxstats']['rx'])
if re.search('{0}'.format(return_ip),output,re.I):
pass
else:
local_stats_list.append(return_ip)
if local_stats_list == []:
log.info('itgen traffic validation for {0} return path -
verified'.format(nat_type))
forward_ip_list = []
return_ip_list = []
else:
log.error('{0} return path itgen verification
failed'.format(nat_type))
flag2 = 0
return flag2
return
flag2,count_dy_Rx,count_dy_in_Rx,count_dy_out_Rx,count_dy_Tx,count_dy_in_Tx,count_dy_out_Tx

*def* *delete_new_dynamic_nat_conf():*
global pool_dy_in_new,acl_name_dy_in_new,pool_name_dy_in_new
direction_in = 'inside'
log.info("Deleting Dynamic NAT new entries")
delete_dynamic_nat_conf(uut2_hdl2, direction_in,
acl_name_dy_in_new,
pool_name_dy_in_new,
)
#remove pool and acl name
log.info('Removing access-list')
deleteACL(uut2_hdl2, log, '-type {0} -name {1}'.format('ip',
acl_name_dy_in_new))
log.info('Unconfiguring pool on UUT')
add_remove_pool(uut2_hdl2, 'unconfigure', pool_dy_in_new)
return 1

On Thu, Apr 27, 2017 at 9:04 PM, Dennis Lee Bieber <wlf...@ix.netcom.com>
wrote:

> On Thu, 27 Apr 2017 19:48:24 +0530, Iranna Mathapati
> <iranna...@gmail.com> declaimed the following:

>
> >Hi Team,
> >
> >I have python code that creates thread.Sometimes it works, sometimes it
> >gets stuck once its finish its execution part. still not yet cannot find
> >any difference between when it works or when it gets stuck. Please help.
> >
> >
> >sniffer1 =
> >threading.Thread(target=validate_traffic_stats_dy,
> args=(FT_item_dy,RT_item_dy,forward_path_list_dy,return_
> path_list_dy,nat_type_list_dy,pkt_dy_1,))
> >
> >sniffer2 =
> >threading.Thread(target=validate_traffic_stats_st,
> args=(FT_item_st,RT_item_st,forward_path_list_st,return_
> path_list_st,nat_type_list_st,pkt_st,))
> >
> >sniffer1.start()
> >
> >sniffer2.start()
> >
> >sniffer3 = threading.Thread(target=delete_new_dynamic_nat_conf)
> >
> >sniffer3.start()
> >
>

> So where is the code for the three thread functions? Where do the
> various argument values come from?
> --
> Wulfraed Dennis Lee Bieber AF6VN
> wlf...@ix.netcom.com HTTP://wlfraed.home.netcom.com/
>
> --
> https://mail.python.org/mailman/listinfo/python-list
>

[Iranna Mathapati]

Thanks Dennis,

On Fri, Apr 28, 2017 at 5:06 AM, Dennis Lee Bieber <wlf...@ix.netcom.com>
wrote:

> On Thu, 27 Apr 2017 22:16:06 +0530, Iranna Mathapati
> <iranna...@gmail.com> declaimed the following:
>

> >Hi Dennis,
> >
> >all function arguments declare as global and pass it to the function.
> >
> >All three thread is completed the execution part and after completion of
> >execution its stuck/hang the programme.
> >
> >*def
> >validate_traffic_stats_st(FT_item,RT_item,forward_path_
> list,return_path_list,nat_type_list,pkt_st)*
> >:
> > global flag1
> > flag1 = 1
> >

> Dangerous usage... Threads should not rely on writing to globals
> unless
> they surround all accesses to the global with a mutual exclusion lock -- OR
> there is only one thread allowed to ever write to the global (a common
> usage is for the main program to set a flag that signals threads to exit,
> and the threads periodically read the flag to see if it has been set).
>
>
> <SNIP>

> > log.error('{0} forward path itgen verification
> >failed'.format(nat_type))
> > flag1 = 0
> > return flag1
>

> threads do not "return" data, they just exit
>
> <SNIP>
> > return
> >flag1,count_st_Rx,count_st_in_Rx,count_st_out_Rx,count_
> st_Tx,count_st_in_Tx,count_st_out_Tx
> >
>
> Ditto -- all those items specified in the return are just being
> dumped
> on the floor.
>
>
> Recommendations: GET RID OF ALL GLOBAL VARIABLES. If you MUST have
> shared global variables, you MUST surround access to the variables with
> mutual exclusion locks -- otherwise you run the risk of counters and lists
> getting out of sync. I'd also suggest compressing sharing down to single
> objects, not half a dozen ad-hoc names. That is:
>
> class St_Record(object):
> def __init__(self):
> self.flag1 = True
> self.count_st_Rx = 0
> self.count_st_in_Rx = 0
> self.count_st_out_Rx = 0
> self.whatever = suitableInitValue
>
> etc. for any other item needed for "st"
>
> Do a similar class for the "dy"
>
>
> Instanciate one instance of each
>
> st_rec = St_Record()
> ...
>
> And inside the relevant threads simply access as
>
> st_rec.count_st_Rx += ...
>
> Assuming these are only used by one thread (and the main program after they
> end) you don't even need to declare them global. Python will find the
> instances at the module level, and one can mutate the instance (ie; the
> contents) without needing to declare global.
>
>
> "Lockup" of the program implies dead-lock, which usually means race
> conditions for the references to data objects between threads, or improper
> use of mutual exclusion.

[Iranna Mathapati]

Hi Dennis,

Two threads(_st and _dy targets) sharing the same user define function at
the same time...This is the reason for hang/stuck?

in both the thread ,we are calling same user define function
(itgen_explorer.startTrafficFlows) and greping the values from return
function ,using it in main function.

[Iranna Mathapati]

Hi Dennis,

Two threads(_st and _dy targets) sharing the same user define function at
the same time...This is the reason for hang/stuck?

in both the thread ,we are calling same user define function
(itgen_explorer.startTrafficFlows) and greping the values from return
function ,using it in main function.

script was hang after completion of the program only .



On Fri, Apr 28, 2017 at 2:42 PM, Iranna Mathapati <iranna...@gmail.com>
wrote:

[Iranna Mathapati]

Hi Dennis,

My requirement is like,i want to send the static and dynamic traffic
together.while traffic sending i want to delete some config from device.

st and dy both the targets want to be run at the same time.



On Fri, Apr 28, 2017 at 6:10 PM, Dennis Lee Bieber <wlf...@ix.netcom.com>
wrote:

> On Fri, 28 Apr 2017 14:42:33 +0530, Iranna Mathapati

> <iranna...@gmail.com> declaimed the following:
>
> >Hi Dennis,
> >

> >Two threads(_st and _dy targets) sharing the same user define function at
> >the same time...This is the reason for hang/stuck?
> >
> >in both the thread ,we are calling same user define function
> >(itgen_explorer.startTrafficFlows) and greping the values from return
> >function ,using it in main function.
> >
>

> While the threads will have their own stacks, meaning function
> local
> data should be separate, if that common function is written anything like
> the st/dy threads (that is, full of "global" declarations) anything could
> be happening.
>
>
> Given that your original sample main logic basically just started
> three
> threads and almost immediately waits for them to complete by using .join(),
> I have to wonder just why you are using threads at all.
>
> Due to the GIL, even on a multi-core processor, only one thread can
> actually do anything at a time (threads work great when they spend most of
> their time waiting for I/O operations to complete, but are terrible if
> trying to do heavy number crunching). If you don't have lots of blocking
> I/O, the thread switching will actually make using threads take more time
> then just calling the functions one after the other.
>
> From your original post, what happens if you replace:

>
> >sniffer1 =
> >threading.Thread(target=validate_traffic_stats_dy,
> args=(FT_item_dy,RT_item_dy,forward_path_list_dy,return_
> path_list_dy,nat_type_list_dy,pkt_dy_1,))
> >sniffer2 =
> >threading.Thread(target=validate_traffic_stats_st,
> args=(FT_item_st,RT_item_st,forward_path_list_st,return_
> path_list_st,nat_type_list_st,pkt_st,))
> >sniffer1.start()
> >sniffer2.start()
> >sniffer3 = threading.Thread(target=delete_new_dynamic_nat_conf)
> >sniffer3.start()

> >sniffer1.join()
> >sniffer2.join()
> >sniffer3.join()
>
> with just the inline calls:
>
> validate_traffic_stats_dy(the arg list)
> validate_traffic_stats_st(its arg list)
> delete_new_dynamic_nat_conf()
>
> That should run them in sequence, with no chance for conflicts
> between
> shared state.
>
> If you still get erratic hanging, you have a logic problem
> somewhere
> else in the program other than a shared state deadlock.