Error restarting rabbitmq

196 views

Skip to first unread message

wil...@kudobuzz.com

unread,

Jan 2, 2017, 3:52:50 AM1/2/17

to rabbitmq-users

Error description:

{could_not_start,rabbit,

{{badmatch,

{error,

{{{function_clause,

[{rabbit_queue_index,journal_minus_segment1,

[{no_pub,no_del,ack},

{{true,

<<215,198,240,235,84,50,31,253,167,226,225,44,

148,144,20,129,0,0,0,0,0,0,0,0,0,0,1,9>>,

<<131,104,6,100,0,13,98,97,115,105,99,95,109,

101,115,115,97,103,101,104,4,100,0,8,114,

101,115,111,117,114,99,101,109,0,0,0,1,47,

100,0,8,101,120,99,104,97,110,103,101,109,

0,0,0,0,108,0,0,0,1,109,0,0,0,28,116,105,

99,116,97,105,108,95,104,105,103,104,101,

115,116,95,101,110,116,105,116,121,95,113,

117,101,117,101,106,104,6,100,0,7,99,111,

110,116,101,110,116,97,60,100,0,4,110,111,

110,101,109,0,0,0,3,16,0,2,100,0,25,114,97,

98,98,105,116,95,102,114,97,109,105,110,

103,95,97,109,113,112,95,48,95,57,95,49,

108,0,0,0,1,109,0,0,1,9,123,34,114,101,113,

117,101,117,101,95,99,111,117,110,116,34,

58,32,50,44,32,34,115,104,111,112,95,105,

100,34,58,32,34,119,81,83,34,44,32,34,100,

111,109,97,105,110,34,58,32,34,116,104,111,

114,112,101,115,101,109,112,111,114,105,

117,109,46,116,105,99,116,97,105,108,46,99,

111,109,34,44,32,34,117,114,108,34,58,32,

34,116,104,111,114,112,101,115,101,109,112,

111,114,105,117,109,46,116,105,99,116,97,

105,108,46,99,111,109,34,44,32,34,105,100,

34,58,32,34,117,117,118,66,34,44,32,34,97,

99,99,111,117,110,116,95,105,100,34,58,32,

49,48,48,49,54,54,56,52,57,51,44,32,34,117,

115,101,114,95,105,100,34,58,32,49,52,52,

51,51,51,48,53,56,50,54,50,55,44,32,34,101,

110,116,105,116,121,95,116,121,112,101,34,

58,32,34,112,114,111,100,117,99,116,115,34,

44,32,34,116,111,107,101,110,34,58,32,34,

97,99,99,101,115,115,116,111,107,101,110,

95,57,71,50,98,57,101,77,78,112,110,51,100,

105,66,112,105,54,114,48,74,108,81,120,110,

114,115,55,103,67,73,34,125,106,109,0,0,0,

16,215,198,240,235,84,50,31,253,167,226,

225,44,148,144,20,129,100,0,4,116,114,117,

101>>},

no_del,no_ack}],

[{file,"src/rabbit_queue_index.erl"},{line,1181}]},

{rabbit_queue_index,'-journal_minus_segment/3-fun-0-',

[{file,"src/rabbit_queue_index.erl"},{line,1158}]},

{array,sparse_foldl_3,7,

[{file,"array.erl"},{line,1690}]},

{array,sparse_foldl_2,9,

[{file,"array.erl"},{line,1684}]},

{rabbit_queue_index,'-recover_journal/1-fun-0-',1,

[{file,"src/rabbit_queue_index.erl"},{line,865}]},

{lists,map,2,[{file,"lists.erl"},{line,1238}]},

{rabbit_queue_index,segment_map,2,

[{file,"src/rabbit_queue_index.erl"},{line,989}]}]},

{gen_server2,call,[<0.207.0>,out,infinity]}},

{child,undefined,msg_store_persistent,

{rabbit_msg_store,start_link,

[msg_store_persistent,

"/var/lib/rabbitmq/mnesia/rabbit@worker",[],

{#Fun<rabbit_queue_index.2.103862237>,

{start,

[{resource,<<"/">>,queue,

<<"storeenvy_order_update_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_order_verification_queue">>},

{resource,<<"/">>,queue,

<<"shop_importation_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_product_verification_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_order_verification_queue">>},

{resource,<<"/">>,queue,

<<"post_update_task_queue">>},

{resource,<<"/">>,queue,

<<"latest_insta_media_queue">>},

{resource,<<"/">>,queue,

<<"bulk_mail_importation_queue">>},

{resource,<<"/">>,queue,

<<"shopify_order_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"shopify_highest_entity_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_product_queue">>},

{resource,<<"/">>,queue,

<<"review_update_task_queue">>},

{resource,<<"/">>,queue,

<<"shopify_product_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_additional_resource_queue">>},

{resource,<<"/">>,queue,

<<"tweet_kudos_importation_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_order_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"tictail_highest_entity_queue">>},

{resource,<<"/">>,queue,

<<"account_marking_queue">>},

{resource,<<"/">>,queue,

<<"tictail_order_queue">>},

{resource,<<"/">>,queue,

<<"rest_storenvy_order_queue">>},

{resource,<<"/">>,queue,

<<"rest_tictail_product_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_order_queue">>},

{resource,<<"/">>,queue,

<<"after_purchase_mail_queue">>},

{resource,<<"/">>,queue,

<<"instagram_comment_kudos_importation_queue">>},

{resource,<<"/">>,queue,

<<"fb_review_from_finder_queue">>},

{resource,<<"/">>,queue,

<<"product_review_migration_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_product_sweep_queue">>},

{resource,<<"/">>,queue,

<<"tweet_importation_queue">>},

{resource,<<"/">>,queue,

<<"tictail_product_sweep_queue">>},

{resource,<<"/">>,queue,

<<"shopify_product_verification_queue">>},

{resource,<<"/">>,queue,

<<"product_importation_queue">>},

{resource,<<"/">>,queue,

<<"tictail_product_verification_queue">>},

{resource,<<"/">>,queue,<<"rest_yelp_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_data_bus">>},

{resource,<<"/">>,queue,

<<"account_migration_queue">>},

{resource,<<"/">>,queue,

<<"facebook_review_importation_queue">>},

{resource,<<"/">>,queue,

<<"tictail_product_queue">>},

{resource,<<"/">>,queue,

<<"instagram_media_rest_call_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_order_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"shopify_product_sweep_queue">>},

{resource,<<"/">>,queue,

<<"instagram_post_kudos_importation_queue">>},

{resource,<<"/">>,queue,

<<"translation_importation_queue">>},

{resource,<<"/">>,queue,

<<"comment_new_task_queue">>},

{resource,<<"/">>,queue,

<<"rest_shopify_product_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_more_product_url_queue">>},

{resource,<<"/">>,queue,

<<"rest_tictail_order_queue">>},

{resource,<<"/">>,queue,

<<"rest_yelp_new_account_queue">>},

{resource,<<"/">>,queue,

<<"fb_post_new_task_queue">>},

{resource,<<"/">>,queue,

<<"review_form_setting_importation_queue">>},

{resource,<<"/">>,queue,

<<"shopify_order_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"comments_update_task_queue">>},

{resource,<<"/">>,queue,

<<"rest_googleplus_queue">>},

{resource,<<"/">>,queue,

<<"new_fb_review_rest_call_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_product_walk_queue">>},

{resource,<<"/">>,queue,

<<"new_tweet_mention_rest_call_queue">>},

{resource,<<"/">>,queue,

<<"shopify_order_sweep_queue">>},

{resource,<<"/">>,queue,

<<"custom_review_migration_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_order_sweep_queue">>},

{resource,<<"/">>,queue,

<<"product_review_from_importation_queue">>},

{resource,<<"/">>,queue,

<<"order_marking_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_more_order_url_queue">>},

{resource,<<"/">>,queue,

<<"rest_bigcommerce_product_queue">>},

{resource,<<"/">>,queue,

<<"new_tweet_favourite_rest_call_queue">>},

{resource,<<"/">>,queue,

<<"shopify_product_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"shopify_order_verification_queue">>},

{resource,<<"/">>,queue,

<<"rest_fb_review_for_new_account_queue">>},

{resource,<<"/">>,queue,

<<"shopify_order_walk_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_order_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"instagram_comment_importation_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_product_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"rest_request_log_queue">>},

{resource,<<"/">>,queue,

<<"woocommerce_product_queue">>},

{resource,<<"/">>,queue,

<<"order_importation_queue">>},

{resource,<<"/">>,queue,

<<"new_tweet_favourite_rest_new_user">>},

{resource,<<"/">>,queue,

<<"rest_shopify_order_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"new_fb_post_rest_call_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_order_sweep_queue">>},

{resource,<<"/">>,queue,

<<"old_insta_media_queue">>},

{resource,<<"/">>,queue,

<<"fb_comment_from_finder_queue">>},

{resource,<<"/">>,queue,

<<"shopify_data_bus_queue">>},

{resource,<<"/">>,queue,

<<"rest_new_account_googleplus_queue">>},

{resource,<<"/">>,queue,

<<"insta_comment_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_product_verification_queue">>},

{resource,<<"/">>,queue,

<<"tagged_media_queue">>},

{resource,<<"/">>,queue,

<<"woocommerce_order_queue">>},

{resource,<<"/">>,queue,

<<"rest_bigcommerce_order_queue">>},

{resource,<<"/">>,queue,

<<"fb_post_from_finder_queue">>},

{resource,<<"/">>,queue,<<"log_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_line_items_queue">>},

{resource,<<"/">>,queue,

<<"tictail_product_walk_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_highest_entity_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_highest_entity_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_product_walk_queue">>},

{resource,<<"/">>,queue,

<<"site_review_migration_queue">>},

{resource,<<"/">>,queue,

<<"insta_comments_for_specific_post_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_product_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"tweet_favourite_queue">>},

{resource,<<"/">>,queue,

<<"tweet_mention_rest_new_user">>},

{resource,<<"/">>,queue,

<<"rest_fb_post_for_new_account_queue">>},

{resource,<<"/">>,queue,

<<"disable_mail_from_mysql_queue">>},

{resource,<<"/">>,queue,

<<"social_review_migration_queue">>},

{resource,<<"/">>,queue,

<<"tictail_order_walk_queue">>},

{resource,<<"/">>,queue,

<<"woocommerce_highest_entity_queue">>},

{resource,<<"/">>,queue,

<<"facebook_comment_kudos_importation_queue">>},

{resource,<<"/">>,queue,

<<"rest_shopify_product_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_product_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"notification_importation_queue">>},

{resource,<<"/">>,queue,

<<"site_review_form_importation_queue">>},

{resource,<<"/">>,queue,

<<"instagram_media_rest_new_user_queue">>},

{resource,<<"/">>,queue,

<<"shop_marking_queue">>},

{resource,<<"/">>,queue,

<<"tweet_mention_queue">>},

{resource,<<"/">>,queue,

<<"fb_entity_url_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_product_update_webhook_queue">>},

{resource,<<"/">>,queue,

<<"new_insta_media_queue">>},

{resource,<<"/">>,queue,

<<"facebook_comment_importation_queue">>},

{resource,<<"/">>,queue,

<<"facebook_post_importation_queue">>},

{resource,<<"/">>,queue,<<"logs_queue">>},

{resource,<<"/">>,queue,

<<"googleplus_data_queue">>},

{resource,<<"/">>,queue,

<<"instagram_account_importation_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_product_queue">>},

{resource,<<"/">>,queue,

<<"rest_shopify_order_queue">>},

{resource,<<"/">>,queue,

<<"new_insta_media_url_queue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_order_walk_queue">>},

{resource,<<"/">>,queue,

<<"twitter_account_importation_queue">>},

{resource,<<"/">>,queue,

<<"tictail_order_sweep_queue">>},

{resource,<<"/">>,queue,

<<"fb_post_new_review_queue">>},

{resource,<<"/">>,queue,

<<"fb_comments_for_specific_post_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_order_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"user_migration_queue">>},

{resource,<<"/">>,queue,

<<"facebook_review_kudos_importation_queue">>},

{resource,<<"/">>,queue,

<<"googleplus_queue">>},

{resource,<<"/">>,queue,

<<"rest_shopify_product_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"tictail_order_verification_queue">>},

{resource,<<"/">>,queue,

<<"rest_storenvy_product_queue">>},

{resource,<<"/">>,queue,

<<"rest_shopify_order_creation_webhook_queue">>},

{resource,<<"/">>,queue,

<<"product_marking_queue">>},

{resource,<<"/">>,queue,

<<"facebook_post_kudos_importation_queue">>},

{resource,<<"/">>,queue,

<<"fb_comment_rest_call_qeue">>},

{resource,<<"/">>,queue,

<<"bigcommerce_data_bus">>},

{resource,<<"/">>,queue,<<"yelp_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_product_sweep_queue">>},

{resource,<<"/">>,queue,

<<"shopify_product_walk_queue">>},

{resource,<<"/">>,queue,

<<"storenvy_order_walk_queue">>},

{resource,<<"/">>,queue,

<<"full_page_review_form_importation_queue">>},

{resource,<<"/">>,queue,

<<"tictail_order_update_queue">>}]}}]},

transient,30000,worker,

[rabbit_msg_store]}}}},

[{rabbit_variable_queue,start_msg_store,2,

[{file,"src/rabbit_variable_queue.erl"},{line,474}]},

{rabbit_variable_queue,start,1,

[{file,"src/rabbit_variable_queue.erl"},{line,456}]},

{rabbit_priority_queue,start,1,

[{file,"src/rabbit_priority_queue.erl"},{line,92}]},

{rabbit_amqqueue,recover,0,

[{file,"src/rabbit_amqqueue.erl"},{line,239}]},

{rabbit,recover,0,[{file,"src/rabbit.erl"},{line,756}]},

{rabbit_boot_steps,'-run_step/2-lc$^1/1-1-',1,

[{file,"src/rabbit_boot_steps.erl"},{line,49}]},

{rabbit_boot_steps,run_step,2,

[{file,"src/rabbit_boot_steps.erl"},{line,49}]},

{rabbit_boot_steps,'-run_boot_steps/1-lc$^0/1-0-',1,

[{file,"src/rabbit_boot_steps.erl"},{line,26}]}]}}

Jean-Sébastien Pédron

unread,

Jan 2, 2017, 4:32:49 AM1/2/17

to rabbitm...@googlegroups.com

On 02.01.2017 09:52, wil...@kudobuzz.com wrote:
> Error description:
> {could_not_start,rabbit,
> {{badmatch,
> {error,
> {{{function_clause,
> [{rabbit_queue_index,journal_minus_segment1,
> [{no_pub,no_del,ack},

Hi!

Could you please provide a bit more details? At least:

- the version of RabbitMQ
- the log files

Thanks!

--
Jean-Sébastien Pédron
Pivotal / RabbitMQ

wil...@kudobuzz.com

unread,

Jan 2, 2017, 5:54:15 AM1/2/17

to rabbitmq-users

Hi,

version: rabbitm 3.6.6

I added the log in the question or do you mean the logs under /var/log/rabbitmq.

the logs added was what I get when i run rabbitmq-server

wil...@kudobuzz.com

unread,

Jan 2, 2017, 5:58:38 AM1/2/17

to rabbitmq-users

This is currently happening in production. I do not want to loose my data.

On Monday, January 2, 2017 at 8:52:50 AM UTC, wil...@kudobuzz.com wrote:

Michael Klishin

unread,

Jan 2, 2017, 6:38:21 AM1/2/17

to rabbitm...@googlegroups.com

We need full logs under /var/log/rabbitmq.

--
You received this message because you are subscribed to the Google Groups "rabbitmq-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to rabbitmq-users+unsubscribe@googlegroups.com.
To post to this group, send email to rabbitmq-users@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

Staff Software Engineer, Pivotal/RabbitMQ

William Obeng

unread,

Jan 2, 2017, 6:40:15 AM1/2/17

to rabbitm...@googlegroups.com

ok a minute

To post to this group, send email to rabbitm...@googlegroups.com.

For more options, visit https://groups.google.com/d/optout.

--
MK

Staff Software Engineer, Pivotal/RabbitMQ

--
You received this message because you are subscribed to a topic in the Google Groups "rabbitmq-users" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/rabbitmq-users/_96tCMxPCTk/unsubscribe.
To unsubscribe from this group and all its topics, send an email to rabbitmq-users+unsubscribe@googlegroups.com.

Michael Klishin

unread,

Jan 2, 2017, 6:40:39 AM1/2/17

to rabbitm...@googlegroups.com

We haven't see this particular error in a pretty long time. Your node fails to recover

a queue index. Deleting queue index file(s) will allow it to start.

However, messages embedded into the index (those smaller than 4096 bytes by default)

will be lost.

--

You received this message because you are subscribed to the Google Groups "rabbitmq-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to rabbitmq-users+unsubscribe@googlegroups.com.

To post to this group, send email to rabbitmq-users@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

wil...@kudobuzz.com

unread,

Jan 2, 2017, 7:08:05 AM1/2/17

to rabbitmq-users

ok. At least I can deal with smaller messages being lost. Where can I find the queue index? Is it inside the mnesia folder in /var/lib/rabbitmq/mnesia/queues? What exactly should I delete.

wil...@kudobuzz.com

unread,

Jan 2, 2017, 7:15:47 AM1/2/17

to rabbitmq-users

link to logs rabbit worker logs and rabbit worker sasl logs

Michael Klishin

unread,

Jan 2, 2017, 7:48:06 AM1/2/17

to rabbitm...@googlegroups.com

Search for journal.jif under the node's database directory.

On 2 Jan 2017, at 20:15, wil...@kudobuzz.com wrote:

link to logs rabbit worker logs and rabbit worker sasl logs

--

You received this message because you are subscribed to the Google Groups "rabbitmq-users" group.

To unsubscribe from this group and stop receiving emails from it, send an email to rabbitmq-user...@googlegroups.com.
To post to this group, send email to rabbitm...@googlegroups.com.

William Obeng

unread,

Jan 2, 2017, 8:05:45 AM1/2/17

to rabbitm...@googlegroups.com

Ok. Thanks

On Jan 2, 2017 12:48 PM, "Michael Klishin" <mkli...@pivotal.io> wrote:

Search for journal.jif under the node's database directory.

On 2 Jan 2017, at 20:15, wil...@kudobuzz.com wrote:

link to logs rabbit worker logs and rabbit worker sasl logs

--
You received this message because you are subscribed to the Google Groups "rabbitmq-users" group.

To unsubscribe from this group and stop receiving emails from it, send an email to rabbitmq-users+unsubscribe@googlegroups.com.
To post to this group, send email to rabbitmq-users@googlegroups.com.

For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to a topic in the Google Groups "rabbitmq-users" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/rabbitmq-users/_96tCMxPCTk/unsubscribe.
To unsubscribe from this group and all its topics, send an email to rabbitmq-users+unsubscribe@googlegroups.com.
To post to this group, send email to rabbitmq-users@googlegroups.com.

William Obeng

unread,

Jan 2, 2017, 8:38:19 AM1/2/17

to rabbitm...@googlegroups.com

thanks. Guys works now. God bless you.

Johan Rhodin

unread,

Oct 22, 2017, 11:00:13 PM10/22/17

to rabbitmq-users

Just a note that this just happened on a 3.6.12 instance with Erlang 19.3.
If logs, .jif-files or something else is of interest, let me know I can send them to support.

To unsubscribe from this group and stop receiving emails from it, send an email to rabbitmq-user...@googlegroups.com.
To post to this group, send email to rabbitm...@googlegroups.com.