Proper use of Tcl_ThreadQueueEvent in a multi-threaded C++ program

[rrr...@gmail.com]

Hi,

I am updating Combat/C++ to allow multi-threaded dispatching. This requires that I queue events to the main thread Tcl interpreter and associated queue from the incoming thread and then wait to be notified when the event is processed by the main Tcl thread before returning the results to my caller.

One problem with my approach is that the Tcl queue code attempts to free the memory for my event and ends up getting an "invalid block" error (see first stack trace below.) Is it allowed to create the Tcl_Event on the original thread? Is there a way around this?

Also, if I sleep for a second before returning control to the TclNotify code, I get a segmentation fault related deallocating thread data. (See second stack trace below.)

I also added my code at the end.

Any pointers :-) would be appreciated!

Thanks!

Rob


--------stack trace with no sleep added ------------------

alloc: invalid block: 0x924410: ef ef 0
#0 0x00002aaaab740875 in raise () from /lib64/libc.so.6
#1 0x00002aaaab741e51 in abort () from /lib64/libc.so.6
#2 0x00002aaaaadb3fa6 in Tcl_PanicVA (format=0x2aaaaae0b6e8 "alloc: invalid block: %p: %x %x %x", argList=0x7fffffffa0e8)
at tcltk-8.5/tcl/unix/../generic/tclPanic.c:103
#3 0x00002aaaaadb4047 in Tcl_Panic (format=0x2aaaaae0b6e8 "alloc: invalid block: %p: %x %x %x")
at tcltk-8.5/tcl/unix/../generic/tclPanic.c:132
#4 0x00002aaaaadd4238 in Ptr2Block (ptr=0x924420 "")
at tcltk-8.5/tcl/unix/../generic/tclThreadAlloc.c:764
#5 0x00002aaaaadd37cb in TclpFree (ptr=0x924420 "")
at tcltk-8.5/tcl/unix/../generic/tclThreadAlloc.c:388
#6 0x00002aaaaad0ccdd in Tcl_Free (ptr=0x924420 "")
at tcltk-8.5/tcl/unix/../generic/tclCkalloc.c:1209
#7 0x00002aaaaadb0462 in Tcl_ServiceEvent (flags=-3)
at tcltk-8.5/tcl/unix/../generic/tclNotify.c:712
#8 0x00002aaaaadb07cd in Tcl_DoOneEvent (flags=-3)
at tcltk-8.5/tcl/unix/../generic/tclNotify.c:980
#9 0x00002aaaaad66d9f in Tcl_VwaitObjCmd (clientData=0x0, interp=0x617f80, objc=2, objv=0x61c658)
at tcltk-8.5/tcl/unix/../generic/tclEvent.c:1349
#10 0x00002aaaaad04ee7 in TclEvalObjvInternal (interp=0x617f80, objc=2, objv=0x61c658,
command=0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff>, length=-1, flags=0)
at tcltk-8.5/tcl/unix/../generic/tclBasic.c:3689
#11 0x00002aaaaad6a19b in TclExecuteByteCode (interp=0x617f80, codePtr=0x65c820)
at tcltk-8.5/tcl/unix/../generic/tclExecute.c:2419
#12 0x00002aaaaadc44d9 in TclObjInterpProcCore (interp=0x617f80, procNameObj=0x772cc0, skip=1,
errorProc=0x2aaaaadc4b6b <MakeProcError>)
at tcltk-8.5/tcl/unix/../generic/tclProc.c:1760
#13 0x00002aaaaadc4413 in TclObjInterpProc (clientData=0x769cd0, interp=0x617f80, objc=2, objv=0x61c4a8)
at tcltk-8.5/tcl/unix/../generic/tclProc.c:1654
#14 0x00002aaaaad04ee7 in TclEvalObjvInternal (interp=0x617f80, objc=2, objv=0x61c4a8,
command=0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff>, length=-1, flags=0)
at tcltk-8.5/tcl/unix/../generic/tclBasic.c:3689
#15 0x00002aaaaad6a19b in TclExecuteByteCode (interp=0x617f80, codePtr=0x7bac50)
at tcltk-8.5/tcl/unix/../generic/tclExecute.c:2419
#16 0x00002aaaaad683d1 in TclCompEvalObj (interp=0x617f80, objPtr=0x712350, invoker=0x61c2a0, word=2)
at tcltk-8.5/tcl/unix/../generic/tclExecute.c:1542
#17 0x00002aaaaad07379 in TclEvalObjEx (interp=0x617f80, objPtr=0x712350, flags=0, invoker=0x61c2a0, word=2)
at tcltk-8.5/tcl/unix/../generic/tclBasic.c:5264
#18 0x00002aaaaad14bad in Tcl_IfObjCmd (dummy=0x0, interp=0x617f80, objc=3, objv=0x61c2f0)
at tcltk-8.5/tcl/unix/../generic/tclCmdIL.c:270
#19 0x00002aaaaad04ee7 in TclEvalObjvInternal (interp=0x617f80, objc=3, objv=0x61c2f0,
command=0x6855a7 "if { [info exists argv0 ] } {\n\tset program [lindex [split $argv0 / ] end]\n\tif { $program == \"ControllerServer.tcl\" } {\n\n\t\tif { ! [info exists argv ] } {\n\t\t\tset argv \"-userid user -channel mychannel -ali"..., length=759, flags=0)
at tcltk-8.5/tcl/unix/../generic/tclBasic.c:3689
~

--------stack trace when the sleep is added----------

#0 PutBlocks (cachePtr=0x92e250, bucket=0, numMove=511)
at tcltk-8.5/tcl/unix/../generic/tclThreadAlloc.c:847
847 lastPtr = lastPtr->nextBlock;
[Current thread is 1 (Thread 0x2aaaaee55700 (LWP 31165))]
(gdb) where
#0 PutBlocks (cachePtr=0x92e250, bucket=0, numMove=511)
at tcltk-8.5/tcl/unix/../generic/tclThreadAlloc.c:847
#1 0x00002aaaaadd344b in TclFreeAllocCache (arg=0x92e250)
at tcltk-8.5/tcl/unix/../generic/tclThreadAlloc.c:242
#2 0x00002aaaaadf2918 in TclpFreeAllocCache (ptr=0x92e250)
at tcltk-8.5/tcl/unix/../unix/tclUnixThrd.c:820
#3 0x00002aaaab27cfb9 in __nptl_deallocate_tsd () from /lib64/libpthread.so.0
#4 0x00002aaaab27e814 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaab7ec67d in clone () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()


--------pertinent routines-------------------

void Combat::DynamicServant::invoke (CORBA::ServerRequest_ptr svr) {
Tcl_ThreadId tid = Tcl_GetCurrentThread();
std::cout << "calling invoke, thread id = " << tid << " operation = " << svr->operation() << "\n";

Tcl_ThreadId mainThread = getMainThread();

CombatEventPointer event = (CombatEventPointer) Tcl_Alloc(sizeof(CombatEventPointer));

// Tcl_Mutex *mutex = new Tcl_Mutex;

// used statics here temporarily for debugging reasons
static Tcl_Mutex mutex;
static Tcl_Condition condition;
static Tcl_Time time={60,0};

event->proc = processEvent;
event->mutex = &mutex;
event->condition = &condition;
event->request = &svr;
event->servant = this;

std::cout << "Locking mutex \n";
Tcl_MutexLock(&mutex);
std::cout << "Queueing event \n";
Tcl_ThreadQueueEvent(mainThread,(Tcl_Event *)event,TCL_QUEUE_TAIL);
Tcl_ThreadAlert(mainThread);
std::cout << "Waiting to be notified by the queue\n";
Tcl_ConditionWait(&condition,&mutex,&time);
Tcl_MutexUnlock(&mutex);
std::cout << "Notified by the queue\n";
}

typedef struct
{
Tcl_EventProc *proc;
struct Tcl_Event *nextPtr;
Tcl_Mutex *mutex;
Tcl_Condition *condition;
Combat::DynamicServant* servant;
CORBA::ServerRequest_ptr * request;
} CombatEvent, *CombatEventPointer;

static int processEvent (Tcl_Event *eventPtr, int flags)
{
std::cout << "processEvent tid = " << Tcl_GetCurrentThread() << " flags = " << flags << "\n";
CombatEventPointer event = (CombatEventPointer)eventPtr;
event->servant->invoke_delegate(*(event->request));
Tcl_MutexLock(event->mutex);
Tcl_ConditionNotify(event->condition);
Tcl_MutexUnlock(event->mutex);
// Tcl_MutexFinalize(event->mutex);
// usleep(1000*1000);
// std::cout << "slept 1000 ms\n";
int status = 1;
return status;
}

[undro...@gmail.com]

Rob,

seems your allocation is wrong

> ...

> CombatEventPointer event = (CombatEventPointer) Tcl_Alloc(sizeof(CombatEventPointer));

> ...

should be sizeof(CombatEvent).

Best,
Christian

[rrr...@gmail.com]

Christian,

You're brilliant! I totally missed that. That fixed all my troubles!
If you're ever in Austin, I'll buy you a beer and a steak!

Thanks for taking the time to review my code!

Rob