My custom engine_finish method does not get called through ENGINE_finish

[Jeff Saremi]

I'm trying to use my custom engine however I cannot get it to clean up nicely.
For the initialization i used the sample in openssl ENGINE(3) documentation.

Here's how it goes:

ENGINE *loadEngine()
{
    ENGINE *e;
    e = ENGINE_by_id(MY_ENGINE_ID);
    if(!e)
        ENGINE_load_my();
    e = ENGINE_by_id(MY_ENGINE_ID); /* try again */
    if(!e)
        /* the engine isn't available */
        return NULL;
    if(!ENGINE_init(e)) {
        /* the engine couldn't initialise, release 'e' */
        ENGINE_free(e);
        return NULL;
    }
    ENGINE_set_default(e, ENGINE_METHOD_ALL);
    return e;
}
void unloadEngine(ENGINE *e)
{
    /* Release the functional reference from ENGINE_init() */
    ENGINE_finish(e);
    /* Release the structural reference from ENGINE_by_id() */
    ENGINE_free(e);
    /* ENGINE_cleanup();  my engine_finish method does not get called unless I include this line */
}
void testKeyGen()
{
   /* some openssl initialization code such as loading ciphers, algorithms, existing engines, establishing dynamic locks and so on */

    ENGINE *e = loadEngine();
    ASSERT(e, "could not create the engine\n");
    EVP_PKEY *pkey = NULL;
    genPKey(e, &pkey); /* some calls to EVP_PKEY_CTX_new_id, EVP_PKEY_keygen and so on */
    ASSERT(pkey, "could not generate PKEY\n");

    /* so far so good */
    if(pkey)
        EVP_PKEY_free(pkey);

    unloadEngine(e);
   /* at the end of this I have unfreed memory (the dynamic lock I created in my engine because my_finish was not called */
}

And here's my engine initialization and deinit routines. I stepped through the code in ENGINE_finish() and realized that the following line (to_return = e->finish(e); ) does not get executed because apparently the ref count is not zero:

********* file openssl/crypto/engine/eng_init.c  **************
int engine_unlocked_finish(ENGINE *e, int unlock_for_handlers)
    {
    if((e->funct_ref == 0) && e->finish)
        {
        if(unlock_for_handlers)
            CRYPTO_w_unlock(CRYPTO_LOCK_ENGINE);
        to_return = e->finish(e);
        if(unlock_for_handlers)
            CRYPTO_w_lock(CRYPTO_LOCK_ENGINE);
        if(!to_return)
            return 0;
...
    }

********* continuation of my code -- my engine init routines  *********

static int my_init(ENGINE *e)
{
    ERR_load_MY_strings();
    my_lock_id = CRYPTO_get_new_dynlockid();
    ...
    return 1;
}
static int my_finish(ENGINE *e)
{
    ERR_unload_MY_strings();
    CRYPTO_destroy_dynlockid(my_lock_id);
    return 1;
}
static int bind_helper(ENGINE *e)
{
...
    if (!ENGINE_set_id(e, MY_ENGINE_ID)
            || !ENGINE_set_name(e, MY_ENGINE_NAME)
            || !ENGINE_set_destroy_function(e, my_destroy)
            || !ENGINE_set_init_function(e, my_init)
            || !ENGINE_set_finish_function(e, my_finish)
...
        return 0;
    return 1;
}
static ENGINE *engine_my(void)
{
    ENGINE *ret = ENGINE_new();
    if (!ret)
        return NULL;
    if (!bind_helper(ret))
    {
        ENGINE_free(ret);
        return NULL;
    }
    return ret;
}
void ENGINE_load_my(void)
{
    ENGINE *toadd = engine_my();
    if (!toadd)
        return;
    ENGINE_add(toadd);
    ENGINE_free(toadd);
    ERR_clear_error();
}

This email contains Morega Systems Inc. Privileged and Confidential information.

[Jeff Saremi]

I fixed the problem. I'm listing it here in case someone else runs into this:

In the loadEngine() I call ENGINE_set_default(). I have to the opposite when unloading my engine.
Unfortunately, I could not find any "ENGINE_unset_default()" or "ENGINE_unregister_all()" so I had to explicitly unregister my method pointers. So my new unloadEngine looks like the following:

void unloadEngine(ENGINE *e)
{
    ENGINE_unregister_pkey_asn1_meths(e);
    ENGINE_unregister_pkey_meths(e);
    ENGINE_unregister_RSA(e);

    ENGINE_remove(e);