Crash on node cleanup, unsure what I did to cause it

[Mike]

I've had this issue for a while now, and kind of stuck where to look to try and figure out what's going on. Hoping someone may have some ideas that I could try.

Background: I have a Qt application (in windows) that embeds an osg viewer running as SingleThreaded for rendering. I wrote an osg plugin which reads a text file that has some hierarchy for displaying OBJ models on the scene. I read the text file, and then place each OBJ model using a PagedLOD.

When browsing around the viewer, and as things page in and out, sometimes I get a crash (not very often) and I can't make sense of the stack trace. It's almost as if I did a double delete or something, however all my nodes are using osg::ref_ptr and I never delete anything manually. Anyone have some thoughts as to what I could investigate based on this stack trace? Unfortunately I can't share the code.

This code used to be based off of OSG 3.0.1, and we've stepped up to 3.6.5 recently but still see the same stack trace. 

Appreciate any ideas anyone has, thanks again.

AddrPC           Params
00007FFE1658F199 00007FFDC978F43C 00000072D2AFF320 0000000000000000  ntdll.dll!RtlReportFatalFailure
00007FFE1658F163 000001938DB0E170 00007FFE165F77F0 0000000000000008  ntdll.dll!RtlReportCriticalFailure
00007FFE16597EE2 0000000000000008 0000000000000000 00000193CD3F0000  ntdll.dll!RtlpHeapHandleError
00007FFE165981CA 00000193CD3F0000 00000193CD3F0000 0000000000000000  ntdll.dll!RtlpHpHeapHandleError
00007FFE1659DE51 0000000000000000 00007FFDC96BA9C0 000001939F8F49F0  ntdll.dll!RtlpLogHeapFailure
00007FFE164B5BF0 000001939CBA7540 00000193CD3F0000 00000193AC452A30  ntdll.dll!RtlpFreeHeapInternal
00007FFE164B47B1 0000000000000001 0000000000000001 FFFFFFFFFFFFFFFE  ntdll.dll!RtlFreeHeap
00007FFE143EF05B 0000000000000001 0000000000000000 00000193CD4F0000  ucrtbase.dll!_free_base
00007FFDA0B15427 0000000000000001 00000193CD3F0000 00000193DBDC2038  osgdb_obj.dll!0x7ffd00035427
00007FFDC96BAE54 00000193AC452858 00000193AC452830 00000193884AE260  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 00000193AC452A30 00000193AC452830 0000019387127B10  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95C0C8D 0000019387127F10 0000000000000001 00000193AB46B2A0  osg161-osg.dll!osg::Viewport::className
00007FFDC96DD538 00000193A81B6690 00000193AB46B2A0 0000000000000000  osg161-osg.dll!osg::StateSet::clear
00007FFDC96DCA76 00000193AB46B2A0 0000000000000001 00000193CD443AC0  osg161-osg.dll!osg::StateSet::~StateSet
00007FFDC96DCDBE 0000000000000001 0000000000000001 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::StateSet::~StateSet
00007FFDC96BAE54 00000193AB46B2A0 00000193A81B6560 00000193A27FD128  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC969C286 00000193A81B6560 000001941ABF3970 00000193A81B6560  osg161-osg.dll!osg::Node::setStateSet
00007FFDC969AEFD 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Node::~Node
00007FFDA0B15395 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_obj.dll!0x7ffd00035395
00007FFDC96BAE54 000001939C4EB6C0 000001939C4EB6C8 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 000001939C4EB6C0 00000193DBDFD9B0 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 000001939C4EB6C8 00000193A27FCF48  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B15355 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_obj.dll!0x7ffd00035355
00007FFDC96BAE54 000001939C4EB500 000001939C4EB508 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 000001939C4EB500 00000193DBDFD9B0 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 000001939C4EB508 00000193A27FD678  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B153D5 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_obj.dll!0x7ffd000353d5
00007FFDC96BAE54 00000193ADE83F60 00000193ADE83F70 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 00000193ADE83F60 00007FFE164B47B1 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 00000193ADE83F70 00000193A27FC598  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B153D5 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_obj.dll!0x7ffd000353d5
00007FFDC96BAE54 000001939C4EBB10 000001939C4EBB18 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 000001939C4EBB10 0000000000000000 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 000001939C4EBB18 00000193A27FD1C8  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0AFE895 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_obj.dll!0x7ffd0001e895
00007FFDC96BAE54 000001939C4EBAC0 000001939C4EBAC8 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 000001939C4EBAC0 000001941ABF3970 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 000001939C4EBAC8 00000193A27FD0D8  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B153D5 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_obj.dll!0x7ffd000353d5
00007FFDC96BAE54 000001939C4EBC00 000001939C4EBC08 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 000001939C4EBC00 00000193DBDFD9B0 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 000001939C4EBC08 0000019300000000  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B68C25 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_objtiles.dll!OGR_SRSNode::`default constructor closure'
00007FFDC96BAE54 000001938F651260 000001938F651268 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 000001938F651260 00007FFE143EF05B FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 000001938F651268 0000019387F481C0  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B68B25 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_objtiles.dll!OGR_SRSNode::`default constructor closure'
00007FFDC96BAE54 0000019388748548 0000019388748550 00000193CD443AC0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC9599E3D 0000019388748548 00000193000006D3 FFFFFFFFFFFFFFFE  osg161-osg.dll!osg::GraphicsContext::ScreenIdentifier::~ScreenIdentifier
00007FFDC95BB0EB 0000000000000001 0000019388748550 000001938F2DF578  osg161-osg.dll!osg::State::Vertex
00007FFDC963E485 0000000000000001 0000000000000001 000001941ABF3970  osg161-osg.dll!osg::Group::~Group
00007FFDA0B68A25 0000000000000001 0000000000000001 00000072D2AFF9D0  osgdb_objtiles.dll!OGR_SRSNode::`default constructor closure'
00007FFDC96BAE54 0000000000000000 0000019399E579E0 00000193DBDFD9E0  osg161-osg.dll!osg::Referenced::signalObserversAndDelete
00007FFDC95A27EC 000001941ABF3EB0 0000019300000000 000001939E98D350  osg161-osg.dll!osg::Referenced::unref
00007FFDD32F9192 00000193E083AEE0 00000193E083AEE0 00000193E083AEF0  osg161-osgDB.dll!osgDB::DatabasePager::DatabaseThread::run
00007FFDF7F42289 00000193DBDC2038 00000193E083AEE0 00000193E083AEF0  ot21-OpenThreads.dll!OpenThreads::SetProcessorAffinityOfCurrentThread
00007FFE14401BB2 0000000000000000 0000000000000000 0000000000000000  ucrtbase.dll!thread_start<unsigned int (__cdecl*)(void *),1>
00007FFE15097034 0000000000000000 0000000000000000 0000000000000000  KERNEL32.DLL!BaseThreadInitThunk
00007FFE164E2651 0000000000000000 0000000000000000 0000000000000000  ntdll.dll!RtlUserThreadStar