rabbitmq-server utilizes whole machine - EBADF (Bad file descriptor)

[Gustav Westling]

Directly after startup, rabbitmq-server allocates 100% CPU and 512MB of RAM.

Erlang (BEAM) emulator version 5.8.5
rabbitmq-server 3.3.5

I have tried to uninstall and delete all remains from rabbitmq-server and installed it again.

After the process is finished everything seems OK for a few seconds until it happens again.

strace-log

close(999163)                           = -1 EBADF (Bad file descriptor)
close(999164)                           = -1 EBADF (Bad file descriptor)
close(999165)                           = -1 EBADF (Bad file descriptor)

....

close(999996)                           = -1 EBADF (Bad file descriptor)
close(999997)                           = -1 EBADF (Bad file descriptor)
close(999998)                           = -1 EBADF (Bad file descriptor)
setsid()                                = 27115
rt_sigprocmask(SIG_UNBLOCK, [CHLD], NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [INT], NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [USR1], NULL, 8) = 0
execve("/bin/sh", ["sh", "-c", "exec /bin/sh -s unix:cmd 2>&1"], [/* 29 vars */]) = 0
brk(0)                                  = 0xc07000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa6c982d000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=46568, ...}) = 0
mmap(NULL, 46568, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa6c9821000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\30\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1815224, ...}) = 0
mmap(NULL, 3929304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa6c924d000
mprotect(0x7fa6c9402000, 2097152, PROT_NONE) = 0
mmap(0x7fa6c9602000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b5000) = 0x7fa6c9602000
mmap(0x7fa6c9608000, 17624, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa6c9608000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa6c9820000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa6c981f000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa6c981e000
arch_prctl(ARCH_SET_FS, 0x7fa6c981f700) = 0
mprotect(0x7fa6c9602000, 16384, PROT_READ) = 0
mprotect(0x619000, 4096, PROT_READ)     = 0
mprotect(0x7fa6c982f000, 4096, PROT_READ) = 0
munmap(0x7fa6c9821000, 46568)           = 0
getpid()                                = 27115
rt_sigaction(SIGCHLD, {0x40f100, ~[RTMIN RT_1], SA_RESTORER, 0x7fa6c92834a0}, NULL, 8) = 0
geteuid()                               = 107
brk(0)                                  = 0xc07000
brk(0xc28000)                           = 0xc28000
getppid()                               = 26773
stat("/var/lib/rabbitmq", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
rt_sigaction(SIGINT, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGINT, {0x40f100, ~[RTMIN RT_1], SA_RESTORER, 0x7fa6c92834a0}, NULL, 8) = 0
rt_sigaction(SIGQUIT, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, ~[RTMIN RT_1], SA_RESTORER, 0x7fa6c92834a0}, NULL, 8) = 0
rt_sigaction(SIGTERM, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGTERM, {SIG_DFL, ~[RTMIN RT_1], SA_RESTORER, 0x7fa6c92834a0}, NULL, 8) = 0
fcntl(2, F_DUPFD, 10)                   = 10
close(2)                                = 0
fcntl(10, F_SETFD, FD_CLOEXEC)          = 0
dup2(1, 2)                              = 2
execve("/bin/sh", ["/bin/sh", "-s", "unix:cmd"], [/* 29 vars */]) = 0
brk(0)                                  = 0x1a00000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f8d62a000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=46568, ...}) = 0
mmap(NULL, 46568, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0f8d61e000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\30\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1815224, ...}) = 0
mmap(NULL, 3929304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f8d04a000
mprotect(0x7f0f8d1ff000, 2097152, PROT_NONE) = 0
mmap(0x7f0f8d3ff000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b5000) = 0x7f0f8d3ff000
mmap(0x7f0f8d405000, 17624, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0f8d405000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f8d61d000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f8d61c000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f8d61b000
arch_prctl(ARCH_SET_FS, 0x7f0f8d61c700) = 0
mprotect(0x7f0f8d3ff000, 16384, PROT_READ) = 0
mprotect(0x619000, 4096, PROT_READ)     = 0
mprotect(0x7f0f8d62c000, 4096, PROT_READ) = 0
munmap(0x7f0f8d61e000, 46568)           = 0
getpid()                                = 27115
rt_sigaction(SIGCHLD, {0x40f100, ~[RTMIN RT_1], SA_RESTORER, 0x7f0f8d0804a0}, NULL, 8) = 0
geteuid()                               = 107
brk(0)                                  = 0x1a00000
brk(0x1a21000)                          = 0x1a21000
getppid()                               = 26773
stat("/var/lib/rabbitmq", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff88b998f8) = -1 ENOTTY (Inappropriate ioctl for device)
rt_sigaction(SIGINT, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, ~[RTMIN RT_1], SA_RESTORER, 0x7f0f8d0804a0}, NULL, 8) = 0
rt_sigaction(SIGQUIT, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, ~[RTMIN RT_1], SA_RESTORER, 0x7f0f8d0804a0}, NULL, 8) = 0
rt_sigaction(SIGTERM, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGTERM, {SIG_DFL, ~[RTMIN RT_1], SA_RESTORER, 0x7f0f8d0804a0}, NULL, 8) = 0
read(0, "(/bin/df -kP /var/lib/rabbitmq/m"..., 8192) = 79
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f0f8d61c9d0) = 27118
wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 27118
--- SIGCHLD (Child exited) @ 0 (0) ---
rt_sigreturn(0x11)                      = 27118
write(1, "\4\n", 2)                     = 2
read(0, "", 8192)                       = 0
exit_group(0)                           = ?
Process 27115 detached

[Michael Klishin]

On 29 August 2014 at 18:49:24, Gustav Westling (gus...@smsgrupp.se) wrote:
> After the process is finished everything seems OK for a few seconds
> until it happens again.

I can't suggest anything definitive from the strace log,
although access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
being repeated over and over looks suspicious.

You seem to be running Erlang R14 or so, please try 17.1.

Also, it would be helpful to know what distro and architecture this is on. 
--
MK

Staff Software Engineer, Pivotal/RabbitMQ