Callback from Windows multimedia timer crashes

[njan...@gmail.com]

I need to call Cython code at a very high rate in Windows, with low jitter.  Using regular threads and delays doesn't work, for reasons I won't go into here.  I've had good luck so far with the Windows multimedia timer using Ctypes, but I can't get the same thing working in Cython.

This code works:

# #####################################

# ctypes_timer.py:

from ctypes import WINFUNCTYPE, windll, c_uint, c_ulong

from ctypes.wintypes import UINT, DWORD

timeproc = WINFUNCTYPE(None, c_uint, c_uint, DWORD, DWORD, DWORD)

timeSetEvent = windll.winmm.timeSetEvent

timeKillEvent = windll.winmm.timeKillEvent

class HighPrecisionPeriodicTimer:

    def __init__(self, interval, callback, resolution=0):

        self.__uDelay = UINT(interval)

        self.__uResolution = UINT(resolution)

        self.__dwUser = c_ulong(0)

        self.__fuEvent = c_uint(True)

        self.__id = None

        self.__callback_func = timeproc(self.__callback)

        self._is_running = False

        self.interval = interval

        self.callback = callback

    def __del__(self):

        try:

            self.stop()

        except:

            pass

    def start(self):

        if not self._is_running:

            self._is_running = True

            self.__id = timeSetEvent(self.__uDelay,

                                     self.__uResolution,

                                     self.__callback_func,

                                     self.__dwUser,

                                     self.__fuEvent)

    def stop(self):

        if self._is_running:

            timeKillEvent(self.__id)

            self._is_running = False

    def __callback(self, uTimerID, uMsg, dwUser, dw1, dw2):

        if self._is_running:

            self.callback()

-----------------------------------

This code doesn't work, I assume I've done something wrong:

# #####################################

# setup.py:

from setuptools import setup

from setuptools.extension import Extension

from Cython.Build import cythonize

from Cython.Distutils import build_ext

cython_timer = Extension(name='cython_timer', sources=["./cython_timer.pyx"], libraries=["Winmm"])

setup(name='cython_timer',

      ext_modules=cythonize([cython_timer]),

      cmdclass={'build_ext': build_ext},

      zip_safe=False)

# #####################################

# cython_timer.pyx:

ctypedef uint32_t MMRESULT

ctypedef uint32_t UINT

ctypedef uint32_t* DWORD_PTR

cdef extern from "Windows.h":

    ctypedef void ( __stdcall *LPTIMECALLBACK)(

        UINT      uTimerID,

        UINT      uMsg,

        DWORD_PTR dwUser,

        DWORD_PTR dw1,

        DWORD_PTR dw2

        )

    cdef MMRESULT timeSetEvent(

        UINT           uDelay,

        UINT           uResolution,

        LPTIMECALLBACK lpTimeProc,

        DWORD_PTR      dwUser,

        UINT           fuEvent

        )

    cdef MMRESULT timeKillEvent(

        UINT uTimerID

        )

cdef class HighPrecisionPeriodicTimer:

    cdef bint      _is_running

    cdef uint32_t  _interval

    cdef uint32_t  _resolution

    cdef uint32_t  _id

    cdef uint32_t* _dwUser

    cdef object    _callback

    def __init__(self, interval, callback, resolution=0):

        if not callable(callback):

            raise TypeError('callback must be callable')

        self._is_running = False

        self._interval   = interval

        self._resolution = resolution

        self._callback   = callback

    def __del__(self):

        try:

            self._stop()

        except:

            pass

    cdef _start(self):

        if not self._is_running:

            self._id = timeSetEvent(self._interval,  # uDelay

                                    self._resolution,  # uResolution

                                    <LPTIMECALLBACK>self._callback_wrapper,  # callback_func

                                    self._dwUser,  # dwUser

                                    1)  # fuEvent, TIME_PERIODIC=1

            self._is_running = True

    cdef _stop(self):

        if self._is_running:

            timeKillEvent(self._id)

            self._is_running = False

    def start(self):

        self._start()

    def stop(self):

        self._stop()

    cdef public void _callback_wrapper(self, uint32_t uTimerID, uint32_t uMsg, uint32_t* dwUser, uint32_t* dw1, uint32_t* dw2):

        if self._is_running:

            self._callback()

# #####################################

# test_timer.py:

import time

from ctypes_timer import HighPrecisionPeriodicTimer

# from cython_timer import HighPrecisionPeriodicTimer

def c():

    print(time.perf_counter())

t = HighPrecisionPeriodicTimer(interval=10, callback=c)

t.start()

time.sleep(1)

t.stop()

The Ctypes version works fine.  When I run the version I tried to convert to Cython, it crashes when start() is called.  I'm sure it's something that I'm not doing right, but I'm stuck at this point.  What am I missing?

[da-woods]

This line can't work:

<LPTIMECALLBACK>self._callback_wrapper,  # callback_func

It's not possible to create a bound C function pointer (because C function pointer simply doesn't have any space to store the self argument). Ctypes does something hacky with runtime-code generation but there there genuinely isn't a way to do it with standard C (hence it isn't possible in Cython either).

The typical pattern is to have a callback function that takes an extra "void*" argument and use that to pass `self` (or whatever else data you need). I'm not sure if your Windows API supports that. Failing that you probably have to use ctypes (you can mix it with Cython if needed).

--

---
You received this message because you are subscribed to the Google Groups "cython-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to cython-users...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/cython-users/bbd29826-b266-4fc1-b563-ce3e2ccaa63cn%40googlegroups.com.

[njan...@gmail.com]

Are you saying it can be done with Cython if I use a plain-old function, thereby getting rid of the "self" argument?  Or are you saying it just can't be done, period?

I tried it as a set of plain-old functions, and it still fails:

# #######################################

cdef uint32_t _timeSetEvent(uint32_t interval, uint32_t resolution, uint32_t* dwUser):

    return timeSetEvent(interval,

                        resolution, 

                        <LPTIMECALLBACK>_callback_wrapper, 

                        dwUser, 

                        1)

cdef _timeKillEvent(uint32_t _id):

    timeKillEvent(_id)

cdef public void _callback_wrapper(uint32_t uTimerID, uint32_t uMsg, uint32_t* dwUser, uint32_t* dw1, uint32_t* dw2):

    print('HERE')

# #######################################

The "self" argument is gone now.  If I don't cast the _callback_wrapper, I get: "Cannot assign type 'void (uint32_t, uint32_t, uint32_t *, uint32_t *, uint32_t *)' to 'LPTIMECALLBACK'".  If I add it, it compiles, but crashes, as it did before.  I would assume there would have to be some way of doing it in Cython, as there are many libraries that require callbacks, no?

[da-woods]

It should work if you use a plain-old function.

I think the reason it's currently failing is because of `__stdcall` - this is part of the function signature and it's important that it matches. It's easily fixed though - you just need to add __stdcall to the signature of the function you define. It should work without a cast.

The other potential issue is the Python GIL - I know nothing about Windows so don't know what thread the high-resolution timer is called from. If it's a different thread then you may need to handle that safely. I'd suggest that you define your function pointer as

ctypedef void ( __stdcall *LPTIMECALLBACK)(

        UINT      uTimerID,

        UINT      uMsg,

        DWORD_PTR dwUser,

        DWORD_PTR dw1,

        DWORD_PTR dw2

        ) nogil

(The "nogil" isn't part of the C signature but it's just extra information for Cython). Your function would similarly need to be declared with "nogil". Depending on what you do inside the function you might need to get back the GIL using a `with gil:` block. This is fine - the important thing would be that the function doesn't have the GIL when it's called.

To view this discussion on the web visit https://groups.google.com/d/msgid/cython-users/2ad6b5ca-2370-4119-a8d3-fb09eec5d1ffn%40googlegroups.com.

[njan...@gmail.com]

It worked!  Thank you so much for your help.  Here's the final version,  for anyone that may stumble across this thread years in the future (myself included):

# ################################################

ctypedef uint32_t MMRESULT

ctypedef uint32_t UINT  # DWORD

ctypedef uint32_t* DWORD_PTR

cdef extern from "Windows.h":

    ctypedef void ( __stdcall *LPTIMECALLBACK)(

        UINT      uTimerID,

        UINT      uMsg,

        DWORD_PTR dwUser,

        DWORD_PTR dw1,

        DWORD_PTR dw2

        ) nogil

    cdef MMRESULT timeSetEvent(

        UINT           uDelay,

        UINT           uResolution,

        LPTIMECALLBACK lpTimeProc,

        DWORD_PTR      dwUser,

        UINT           fuEvent

        )

    cdef MMRESULT timeKillEvent(

        UINT uTimerID

        )

cdef uint32_t _timeSetEvent(uint32_t interval, uint32_t resolution, uint32_t* dwUser):

    return timeSetEvent(interval,

                        resolution,

                        _callback_wrapper,

                        dwUser,

                        1)

cdef _timeKillEvent(uint32_t _id):

    timeKillEvent(_id)

cdef public void __stdcall _callback_wrapper(uint32_t uTimerID, uint32_t uMsg, uint32_t* dwUser, uint32_t* dw1, uint32_t* dw2) nogil:

    with gil:

        print('Call your actual callback here')

# ################################################

Creating a class that calls these plain-old functions is left as an exercise to the reader.  If you use the class as a singleton, then just store your 'real' callback callable in a global variable.  If you need more instances than that, maybe store all of the callables in a global variable dict, using the key the returned IDs from timeSetEvent() as the key.  They're not indexed 0, 1, 2, n+1, etc, so I assume you'd have to use a hash table to be safe.

Thanks again, da_woods, this was a bit out of my wheel-house, I'm grateful for the support on this group.