Python non blocking multi-client service

[dimao]

I am trying to implement multi-client service. The service must be able to:

connect to the server;
send/receive messages;
wait until a new message will be received
*** Per each client and non blocking

My current code takes almost 99% CPU usage. Here is the main part of my code :


PORT = 33123
HOST = '127.0.0.1'


import asyncio
import os

@asyncio.coroutine
def tcp_echo_client(offset):

def send(offset):
MSG = """{"ClientId":"%s", % (str(offset))"}"""

print("> " + MSG)
writer.write((MSG).encode("utf-8"))


def recv():
msgback = (yield from reader.readline()).decode("utf-8").rstrip()
print("< " + msgback)
return msgback

reader, writer = yield from asyncio.open_connection(HOST, port=PORT)


print(reader)
print('Waiting 3 sec for response...')

while True:
response = yield from asyncio.wait_for(reader.readline(), timeout=5.0)
print(response)
send(offset)

yield from asyncio.sleep(0.5)

@asyncio.coroutine
def do_work(task_name, work_queue):
while not work_queue.empty():
queue_item = yield from work_queue.get()
print('{0} grabbed item: {1}'.format(task_name, queue_item))
asyncio.Task(tcp_echo_client(offset=queue_item))
yield from asyncio.sleep(0.1)


if __name__ == "__main__":
q = asyncio.Queue()

for x in range(100):
q.put_nowait(x)

print(q)

loop = asyncio.get_event_loop()

tasks = [
asyncio.async(do_work('task1', q)),
asyncio.async(do_work('task2', q)),
asyncio.async(do_work('task3', q)),
asyncio.async(do_work('task4', q)),
asyncio.async(do_work('task5', q)),
asyncio.async(do_work('task6', q))
]

loop.run_until_complete(asyncio.wait(tasks))
loop.run_forever()
loop.close()

[Chris Angelico]

On Tue, Aug 23, 2016 at 11:08 PM, dimao <dima....@cell-buddy.com> wrote:
> My current code takes almost 99% CPU usage. Here is the main part of my code :
>
>
> PORT = 33123
> HOST = '127.0.0.1'
>
>
> import asyncio
> import os
>
> @asyncio.coroutine
> def tcp_echo_client(offset):
>
> def send(offset):
> MSG = """{"ClientId":"%s", % (str(offset))"}"""
>
> print("> " + MSG)
> writer.write((MSG).encode("utf-8"))
>
>
> def recv():
> msgback = (yield from reader.readline()).decode("utf-8").rstrip()
> print("< " + msgback)
> return msgback
>
> reader, writer = yield from asyncio.open_connection(HOST, port=PORT)
>
>
> print(reader)
> print('Waiting 3 sec for response...')
>
> while True:
> response = yield from asyncio.wait_for(reader.readline(), timeout=5.0)
> print(response)
> send(offset)
>
> yield from asyncio.sleep(0.5)
>

Can you post your actual code, please? I'm not sure that this code is
runnable - at least, not with this indentation. And if I have to fix
your indentation to try to figure out what's nested inside what, I
can't help with your CPU usage problem.

ChrisA

[dima....@cell-buddy.com]

UPDATED :

PORT = 33123
HOST = '127.0.0.1'

LASTLINE = '#'

import asyncio
import os


@asyncio.coroutine
def tcp_echo_client(offset):

def send(offset):

MSG = '{'+"ClientId"+':'+"%s" % (str(offset))+'}'

print("> " + MSG)
writer.write((MSG ).encode("utf-8"))
yield from writer.drain()

def recv():
msgback = (yield from reader.readline()).decode("utf-8").rstrip()
print("< " + msgback)
return msgback

reader, writer = yield from asyncio.open_connection(HOST, port=PORT)


print(reader)
print('Waiting 3 sec for response...')

while True:

try:
response = yield from asyncio.wait_for(reader.readline(), timeout=100000.0)
print(response)

if str(response).find('GetClientInfo') > 0:
send(int(offset))

yield from asyncio.sleep(0.5)
except:
print ('Error')

@asyncio.coroutine
def do_work(task_name, work_queue):
while not work_queue.empty():
queue_item = yield from work_queue.get()
print('{0} grabbed item: {1}'.format(task_name, queue_item))
asyncio.Task(tcp_echo_client(offset=queue_item))
yield from asyncio.sleep(0.1)



if __name__ == "__main__":
q = asyncio.Queue()

for x in range(1):

q.put_nowait(x)

print(q)

loop = asyncio.get_event_loop()

tasks = [
asyncio.async(do_work('task1', q)),
asyncio.async(do_work('task2', q)),
asyncio.async(do_work('task3', q)),
asyncio.async(do_work('task4', q)),
asyncio.async(do_work('task5', q)),
asyncio.async(do_work('task6', q))
]

loop.run_until_complete(asyncio.wait(tasks))
loop.run_forever()
loop.close()

Thanks

[Chris Angelico]

On Tue, Aug 23, 2016 at 11:39 PM, <dima....@cell-buddy.com> wrote:
> except:
> print ('Error')
>

Don't do this.

ChrisA

[dima....@cell-buddy.com]

On Tuesday, August 23, 2016 at 4:09:07 PM UTC+3, dimao wrote:

I did that only for the debug reasons :-)

[dima....@cell-buddy.com]

On Tuesday, August 23, 2016 at 4:09:07 PM UTC+3, dimao wrote:

Can any one help me ?

[Zentrader]

I don't know anything about asyncio, but multiprocessing would be my tool of choice. The first example should be enough for what you want to do at https://pymotw.com/2/multiprocessing/basics.html

[INADA Naoki]

>
> I did that only for the debug reasons :-)

That's bad for debugging too.
Real Exception and Stacktrace are far better than print('Error')
on all time. Never do it even for debugging.

[sohca...@gmail.com]

On Tue, Aug 23, 2016 at 11:39 PM, <dima....@cell-buddy.com> wrote:

> On Tuesday, August 23, 2016 at 6:42:53 AM UTC-7, Chris Angelico wrote:
> > On Tuesday, August 23, 2016 at 4:09:07 PM UTC+3, dimao wrote:

> > > except:
> > > print ('Error')
> >
> >
> > Don't do this.
> >
> > ChrisA
>

> I did that only for the debug reasons :-)

That makes it even worse. If you're trying to debug something, you want as much information as you can get, and you're throwing it all away.

If you want your program to keep going no matter what happened, at *least* print the stack trace, and not just "Error".