Cowboy Stream Errors in Cluster Operator

55 views
Skip to first unread message

Ralph Otowo

unread,
Jan 13, 2023, 2:09:55 PM1/13/23
to rabbitmq-users
I recently deployed the RabbitMQ cluster operator in our cluster and tried to create a RabbitMQ Cluster. I'm getting a weird error in the logs after the pods start up. The log file is attached. Does anyone know what the issue might be? My searches aren't yielding helpful results.

2023-01-13 18:39:03.148198+00:00 [info] <0.662.0> node 'rab...@rabbitmq-operator-server-2.rabbitmq-operator-nodes.rabbitmq' up
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>   crasher:
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     initial call: cowboy_stream_h:request_process/3
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     pid: <0.1624.0>
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     registered_name: []
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     exception error: bad argument
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in operator  ++/2
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>          called as {badrpc,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {'EXIT',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                      {badarg,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                       [{ets,match_object,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                         [rabbit_listener_ets,'_'],
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                         [{error_info,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                           #{cause => id,module => erl_stdlib_errors}}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                        {ets,tab2list,1,[{file,"ets.erl"},{line,771}]}]}}}
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                    ++
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                    [{listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               http,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               15672,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               [{cowboy_opts,[{sendfile,false}]},{port,15672}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               'http/prometheus',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               15692,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               [{cowboy_opts,[{sendfile,false}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {port,15692},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {protocol,'http/prometheus'}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               clustering,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               25672,[]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               amqp,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               5672,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               [{backlog,128},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {nodelay,true},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {linger,{true,0}},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {exit_on_close,false}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               http,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               15672,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               [{cowboy_opts,[{sendfile,false}]},{port,15672}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               'http/prometheus',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               15692,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               [{cowboy_opts,[{sendfile,false}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {port,15692},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {protocol,'http/prometheus'}]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               clustering,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               25672,[]},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                     {listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               amqp,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               "rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               {0,0,0,0,0,0,0,0},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               5672,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                               [{backlog,128},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {nodelay,true},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {linger,{true,0}},
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                                {exit_on_close,false}]}]
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from lists:append/1 (lists.erl, line 155)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from rabbit_mgmt_wm_overview:listeners/0 (rabbit_mgmt_wm_overview.erl, line 111)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from rabbit_mgmt_wm_overview:to_json/2 (rabbit_mgmt_wm_overview.erl, line 66)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from cowboy_rest:call/3 (src/cowboy_rest.erl, line 1575)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from cowboy_rest:set_resp_body/2 (src/cowboy_rest.erl, line 1464)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from cowboy_rest:upgrade/4 (src/cowboy_rest.erl, line 284)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>       in call from cowboy_stream_h:execute/3 (src/cowboy_stream_h.erl, line 300)
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     ancestors: [<0.1623.0>,<0.835.0>,<0.828.0>,<0.827.0>,<0.825.0>,
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>                   rabbit_web_dispatch_sup,<0.814.0>]
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     message_queue_len: 0
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     messages: []
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     links: [<0.1623.0>]
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     dictionary: []
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     trap_exit: false
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     status: running
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     heap_size: 4185
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     stack_size: 28
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>     reductions: 3416
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>   neighbours:
2023-01-13 18:39:03.692368+00:00 [error] <0.1624.0>
2023-01-13 18:39:03.694005+00:00 [error] <0.1623.0> Ranch listener {acceptor,{0,0,0,0,0,0,0,0},15672}, connection process <0.1623.0>, stream 1 had its request process <0.1624.0> exit with reason badarg and stacktrace [{erlang,'++',[{badrpc,{'EXIT',{badarg,[{ets,match_object,[rabbit_listener_ets,'_'],[{error_info,#{cause => id,module => erl_stdlib_errors}}]},{ets,tab2list,1,[{file,"ets.erl"},{line,771}]}]}}},[{listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',http,"rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},15672,[{cowboy_opts,[{sendfile,false}]},{port,15672}]},{listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq','http/prometheus',"rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},15692,[{cowboy_opts,[{sendfile,false}]},{port,15692},{protocol,'http/prometheus'}]},{listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',clustering,"rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},25672,[]},{listener,'rab...@rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq',amqp,"rabbitmq-operator-server-1.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},5672,[{backlog,128},{nodelay,true},{linger,{true,0}},{exit_on_close,false}]},{listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',http,"rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},15672,[{cowboy_opts,[{sendfile,false}]},{port,15672}]},{listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq','http/prometheus',"rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},15692,[{cowboy_opts,[{sendfile,false}]},{port,15692},{protocol,'http/prometheus'}]},{listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',clustering,"rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},25672,[]},{listener,'rab...@rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq',amqp,"rabbitmq-operator-server-0.rabbitmq-operator-nodes.rabbitmq.svc.cluster.local",{0,0,0,0,0,0,0,0},5672,[{backlog,128},{nodelay,true},{linger,{true,0}},{exit_on_close,false}]}]],[{error_info,#{module => erl_erts_errors}}]},{lists,append,1,[{file,"lists.erl"},{line,155}]},{rabbit_mgmt_wm_overview,listeners,0,[{file,"rabbit_mgmt_wm_overview.erl"},{line,111}]},{rabbit_mgmt_wm_overview,to_json,2,[{file,"rabbit_mgmt_wm_overview.erl"},{line,66}]},{cowboy_rest,call,3,[{file,"src/cowboy_rest.erl"},{line,1575}]},{cowboy_rest,set_resp_body,2,[{file,"src/cowboy_rest.erl"},{line,1464}]},{cowboy_rest,upgrade,4,[{file,"src/cowboy_rest.erl"},{line,284}]},{cowboy_stream_h,execute,3,[{file,"src/cowboy_stream_h.erl"},{line,300}]}]
2023-01-13 18:39:03.694005+00:00 [error] <0.1623.0>
2023-01-13 18:39:04.067541+00:00 [info] <0.662.0> rabbit on node 'rab...@rabbitmq-operator-server-2.rabbitmq-operator-nodes.rabbitmq' up

Reply all
Reply to author
Forward
0 new messages