Handling alloc failures in Java_com_sun_jna_Native_registerMethod

[David Turner]

Hi,

I was looking into a situation in which Elasticsearch crashes with a SIGSEGV after calling JNA's Java_com_sun_jna_Native_registerMethod when it has a bad config and is running in an OS that's hardened in some way. Although I think we probably do want to exit the process in these situations, the abrupt segfault means that we can't really catch the problem and translate it to something that guides the user towards resolution. My comment on the Elasticsearch issue is here:

https://github.com/elastic/elasticsearch/issues/73309#issuecomment-907597693 for details.

I looked through the JNA source without really knowing much about what it does and saw a handful of places where we're not checking return codes. I believe the actual problem is that ffi_closure_alloc is returning NULL in these configs, but there are other allocations that go unchecked too in that function. I've copied the function source from `master` and added annotations of the lines that I think could yield unexpected NULLs sometimes, see below. Does this seem like a plausible explanation of our problem or am I looking in completely the wrong place?

Cheers,

David

JNIEXPORT jlong JNICALL

Java_com_sun_jna_Native_registerMethod(JNIEnv *env, jclass UNUSED(ncls),

                                       jclass cls, jstring name,

                                       jstring signature,

                                       jintArray conversions,

                                       jlongArray closure_atypes,

                                       jlongArray atypes,

                                       jint rconversion,

                                       jlong closure_return_type,

                                       jlong return_type,

                                       jobject closure_method,

                                       jlong function, jint cc,

                                       jboolean throw_last_error,

                                       jobjectArray to_native,

                                       jobject from_native,

                                       jstring encoding)

{

  int argc = atypes ? (*env)->GetArrayLength(env, atypes) : 0;

  const char* cname = newCStringUTF8(env, name);            // <--(DCT)-- this might return NULL

  const char* sig = newCStringUTF8(env, signature);         // <--(DCT)-- this might return NULL

  void *code;

  void *closure;

  method_data* data = malloc(sizeof(method_data));          // <--(DCT)-- this might return NULL

  ffi_cif* closure_cif = &data->closure_cif;

  int status;

  int i;

  int abi = cc == CALLCONV_C ? FFI_DEFAULT_ABI : cc;

  ffi_type* rtype = (ffi_type*)L2A(return_type);

  ffi_type* closure_rtype = (ffi_type*)L2A(closure_return_type);

  jlong* types = atypes ? (*env)->GetLongArrayElements(env, atypes, NULL) : NULL;

  jlong* closure_types = closure_atypes ? (*env)->GetLongArrayElements(env, closure_atypes, NULL) : NULL;

  jint* cvts = conversions ? (*env)->GetIntArrayElements(env, conversions, NULL) : NULL;

#if defined(_WIN32) && !defined(_WIN64) && !defined(_WIN32_WCE)

  if (cc == CALLCONV_STDCALL) abi = FFI_STDCALL;

#endif

  if (!(abi > FFI_FIRST_ABI && abi < FFI_LAST_ABI)) {

    char msg[MSG_SIZE];

    snprintf(msg, sizeof(msg), "Invalid calling convention %d", abi);

    throwByName(env, EIllegalArgument, msg);

    status = FFI_BAD_ABI;

    goto cleanup;                                           // <--(DCT)-- cleanup frees uninitialized data->arg_types and data->flags

  }

  data->throw_last_error = throw_last_error;

  data->arg_types = malloc(sizeof(ffi_type*) * argc);                        // <--(DCT)-- this might return NULL

  data->closure_arg_types = malloc(sizeof(ffi_type*) * (argc + 2));          // <--(DCT)-- this might return NULL

  data->closure_arg_types[0] = &ffi_type_pointer;

  data->closure_arg_types[1] = &ffi_type_pointer;

  data->closure_method = NULL;

  data->flags = cvts ? malloc(sizeof(jint)*argc) : NULL;                     // <--(DCT)-- this might return NULL (maybe ok?)

  data->rflag = rconversion;

  data->to_native = NULL;

  data->from_native = from_native ? (*env)->NewWeakGlobalRef(env, from_native) : NULL;

  data->encoding = newCStringUTF8(env, encoding);                            // <--(DCT)-- this might return NULL

  for (i=0;i < argc;i++) {

    data->closure_arg_types[i+2] = (ffi_type*)L2A(closure_types[i]);

    data->arg_types[i] = (ffi_type*)L2A(types[i]);

    if (cvts) {

      data->flags[i] = cvts[i];

      // Type mappers only apply to non-primitive arguments

      if (cvts[i] == CVT_TYPE_MAPPER

          || cvts[i] == CVT_TYPE_MAPPER_STRING

          || cvts[i] == CVT_TYPE_MAPPER_WSTRING) {

        if (!data->to_native) {

          data->to_native = calloc(argc, sizeof(jweak));

        }

        data->to_native[i] = (*env)->NewWeakGlobalRef(env, (*env)->GetObjectArrayElement(env, to_native, i));

      }

    }

  }

  if (types) (*env)->ReleaseLongArrayElements(env, atypes, types, 0);

  if (closure_types) (*env)->ReleaseLongArrayElements(env, closure_atypes, closure_types, 0);

  if (cvts) (*env)->ReleaseIntArrayElements(env, conversions, cvts, 0);

  data->fptr = L2A(function);

  data->closure_method = (*env)->NewGlobalRef(env, closure_method);

  status = ffi_prep_cif(closure_cif, abi, argc+2, closure_rtype, data->closure_arg_types);

  if (ffi_error(env, "Native method mapping", status)) {

    goto cleanup;

  }

  status = ffi_prep_cif(&data->cif, abi, argc, rtype, data->arg_types);

  if (ffi_error(env, "Native method setup", status)) {

    goto cleanup;

  }

  closure = ffi_closure_alloc(sizeof(ffi_closure), &code);                            // <--(DCT)-- this might return NULL

  status = ffi_prep_closure_loc(closure, closure_cif, dispatch_direct, data, code);

  if (status != FFI_OK) {

    throwByName(env, EError, "Native method linkage failed");

    goto cleanup;

  }

  {

    JNINativeMethod m = { (char*)cname, (char*)sig, code };

    (*env)->RegisterNatives(env, cls, &m, 1);

  }

 cleanup:

  if (status != FFI_OK) {

    free(data->arg_types);

    free(data->flags);

    free(data);

    data = NULL;

  }

  free((void *)cname);

  free((void *)sig);

  return A2L(data);

}

Java_com_sun_jna_Native_registerMethod