SimpleMessageListenerContainer - Failed to check/redeclare auto-delete queue(s).

[dimple dass]

We are upgrading RabbitMQ to  RabbitMQ 3.9.7Erlang 24.0.5 from  RabbitMQ 3.8.7Erlang 22.0.7 on server ; upgraded to spring boot 2.5.12; with  <spring.ampqp.version>2.2.9.RELEASE</spring.ampqp.version>

Please see below code snippets...please help me figure out what the issue is ...
The below code works fine if I run locally on my laptop but fails when I push it to dev environment.

I deleted existing queues and restarted the application but that didnt help.
---------------------------------------

Here are my dependencies in pom file:
<parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.5.12</version>
    </parent>

 <spring.ampqp.version>2.2.9.RELEASE</spring.ampqp.version>

 <dependency>
            <groupId>org.springframework.amqp</groupId>
            <artifactId>spring-amqp</artifactId>
            <version>${spring.ampqp.version}</version>
        </dependency>
         <dependency>
            <groupId>org.springframework.amqp</groupId>
            <artifactId>spring-rabbit</artifactId>
            <version>${spring.ampqp.version}</version>
        </dependency>

---------------------------------------

Here is my Rabbit config file:

@EnableRabbit
@Configuration
public class RabbitConfiguration {

    @Value("${amqp.testApiRequest}")
    private String testApiRequest;
       
    @Value("${amqp.testApiResponse}")
    private String testApiResponse;
   
    @Value("${amqp.testOutRequest}")
    private String testOutRequest;
   
    @Value("${amqp.testOutResponse}")
    private String testOutResponse;

    @Value("${amqp.exchange}")
    private String exchange;

    @Value("${amqp.username}")
    private String username;

    @Value("${amqp.password}")
    private String password;

    @Value("${amqp.host}")
    private String host;

    @Value("${amqp.vhost}")
    private String virtualHost;
   
    @Autowired
    private OutboundResponseMessageHandler outboundResponseMessageHandler;
   
   
    @Autowired
    private OutboundAPIMessageHandler outboundAPIMessageHandler;

    @Bean
    public DirectExchange exchange() {
        return new DirectExchange(exchange);
    }
   
    @Bean
    Queue testApiRequestQueue() {
        return new Queue(testApiRequest, true);
    }

    @Bean
    Queue testApiResponseQueue() {
        return new Queue(testApiResponse, true);
    }
    @Bean
    Queue testOutRequestQueue() {
        return new Queue(testOutRequest, true);
    }

    @Bean
    Queue testOutResponseQueue() {
        return new Queue(testOutResponse, true);
    }
   
    @Bean
    Binding testApiRequestBinding(DirectExchange exchange) {
        return BindingBuilder.bind(testApiRequestQueue()).to(exchange).with(testApiRequest);
    }

    @Bean
    Binding testApiResponseBinding(DirectExchange exchange) {
        return BindingBuilder.bind(testApiResponseQueue()).to(exchange).with(testApiResponse);
    }

    @Bean
    Binding testOutRequestBinding(DirectExchange exchange) {
        return BindingBuilder.bind(testOutRequestQueue()).to(exchange).with(testOutRequest);
    }

    @Bean
    Binding testOutResponseBinding(DirectExchange exchange) {
        return BindingBuilder.bind(testOutResponseQueue()).to(exchange).with(testOutResponse);
    }
 
  @Bean("ConnectionFactory")
    public ConnectionFactory connectionFactory() {
        CachingConnectionFactory connectionFactory = new CachingConnectionFactory();
        connectionFactory.setVirtualHost(virtualHost);
        connectionFactory.setHost(host);
        connectionFactory.setUsername(username);
        connectionFactory.setPassword(password);
        return connectionFactory;
    }

    @Bean("RabbitTemplate")
    public RabbitTemplate rabbitTemplate(@Autowired @Qualifier("ConnectionFactory") ConnectionFactory connectionFactory) {
        RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory);
        rabbitTemplate.setMessageConverter(xmlMessageConverter() );
        return rabbitTemplate;
    }

   @Bean("Jackson2XmlMessageConverter")
    public Jackson2XmlMessageConverter xmlMessageConverter() {
        return new Jackson2XmlMessageConverter();
    }
      @Bean
      public AmqpAdmin amqpAdmin() {
        return new  RabbitAdmin(connectionFactory());
    }
   
      @RabbitListener(queues = "${amqp.testApiResponse}", id ="testApiResponseListener")      
      @RabbitHandler
      @ServiceActivator
      public void testApiResponse(Message message) throws Exception {
          outboundAPIMessageHandler.handleMessage(message);
          }
     
      @RabbitListener(queues = "${amqp.testOutResponse}", id ="testOutResponseListener")      
     @RabbitHandler
     @ServiceActivator
     public void testOutResponse(Message message) throws Exception {
          outboundResponseMessageHandler.handleMessage(message);
          }
     
    @Bean
    public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory() {
        final SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
        factory.setConnectionFactory(connectionFactory());
        factory.setErrorHandler(errorHandler());
        factory.setAcknowledgeMode(AcknowledgeMode.AUTO);
        return factory;
    }
   
    @Bean
    public ErrorHandler errorHandler() {
        return new ConditionalRejectingErrorHandler(new MyFatalExceptionStrategy());
    }

    public static class MyFatalExceptionStrategy extends ConditionalRejectingErrorHandler.DefaultExceptionStrategy {

        private final Logger logger = LogManager.getLogger(getClass());

        @Override
        public boolean isFatal(Throwable t) {
            if (t instanceof ListenerExecutionFailedException) {
                ListenerExecutionFailedException lefe = (ListenerExecutionFailedException) t;
                logger.error("Failed to process inbound message from queue "
                        + lefe.getFailedMessage().getMessageProperties().getConsumerQueue()
                        + "; failed message: " + lefe.getFailedMessage(), t);
            }
            return super.isFatal(t);
        }

    }
   
    @Bean("rabbitApiRequestWriter")
    public QueueWriter rabbitApiRequestWriter(@Autowired @Qualifier("ConnectionFactory") ConnectionFactory cf) {
       
        return new RabbitQueueWriter(cf, testApiRequest);
    }
   
    @Bean("rabbitApiResponseWriter")
    public QueueWriter rabbitApiResponseWriter(@Autowired @Qualifier("ConnectionFactory") ConnectionFactory cf) {
       
        return new RabbitQueueWriter(cf, testApiResponse);
    }
    @Bean("rabbitOutRequestWriter")
    public QueueWriter rabbitOutRequestWriter(@Autowired @Qualifier("ConnectionFactory") ConnectionFactory cf) {
       
        return new RabbitQueueWriter(cf, testOutRequest);
    }
    @Bean("rabbitOutResponseWriter")
    public QueueWriter rabbitOutResponseWriter(@Autowired @Qualifier("ConnectionFactory") ConnectionFactory cf) {
       
        return new RabbitQueueWriter(cf, testOutResponse);
    }
}

-------------------
application.yml

amqp:
  testApiRequest: 'test.api.request'
  testApiResponse: 'test.api.response'
  testOutRequest: 'test.out.request'
  testOutResponse: 'test.out.response'
  host: testdev01
  username: admin
  password: xxxx
  port: 6672
  vhost: /

----------------

Error:

ERROR o.s.a.r.l.SimpleMessageListenerContainer - Failed to check/redeclare auto-delete queue(s).
org.springframework.amqp.AmqpConnectException: java.net.ConnectException: Connection refused (Connection refused)
    at org.springframework.amqp.rabbit.support.RabbitExceptionTranslator.convertRabbitAccessException(RabbitExceptionTranslator.java:61)
    at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.createBareConnection(AbstractConnectionFactory.java:524)
    at org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createConnection(CachingConnectionFactory.java:751)
    at org.springframework.amqp.rabbit.connection.ConnectionFactoryUtils.createConnection(ConnectionFactoryUtils.java:214)
    at org.springframework.amqp.rabbit.core.RabbitTemplate.doExecute(RabbitTemplate.java:2089)
    at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:2062)
    at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:2042)
    at org.springframework.amqp.rabbit.core.RabbitAdmin.getQueueInfo(RabbitAdmin.java:407)
    at org.springframework.amqp.rabbit.core.RabbitAdmin.getQueueProperties(RabbitAdmin.java:391)
    at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.attemptDeclarations(AbstractMessageListenerContainer.java:1842)
    at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.redeclareElementsIfNecessary(AbstractMessageListenerContainer.java:1823)
    at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.initialize(SimpleMessageListenerContainer.java:1349)
    at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.run(SimpleMessageListenerContainer.java:1195)
    at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.net.ConnectException: Connection refused (Connection refused)
    at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
    at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:399)
    at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:242)
    at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:224)
    at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    at java.base/java.net.Socket.connect(Socket.java:609)
    at com.rabbitmq.client.impl.SocketFrameHandlerFactory.create(SocketFrameHandlerFactory.java:60)
    at com.rabbitmq.client.ConnectionFactory.newConnection(ConnectionFactory.java:1220)
    at com.rabbitmq.client.ConnectionFactory.newConnection(ConnectionFactory.java:1170)
    at com.rabbitmq.client.ConnectionFactory.newConnection(ConnectionFactory.java:1338)
    at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.connectHostPort(AbstractConnectionFactory.java:569)
    at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.connect(AbstractConnectionFactory.java:536)
    at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.createBareConnection(AbstractConnectionFactory.java:487)
    ... 12 common frames omitted

WARN  o.s.a.r.l.SimpleMessageListenerContainer - Consumer raised exception, processing can restart if the connection factory supports it. Exception summary: org.springframework.amqp.AmqpConnectException: java.net.ConnectException: Connection refused (Connection refused)