[esb-issues] [JBoss JIRA] Reopened: (JBESB-2184) Endless loop during cluster failover, caused by JmsCourier.jmsConnectRetry

Endless loop during cluster failover, caused by JmsCourier.jmsConnectRetry -------------------------------------------------------------------------- Key: JBESB-2184 URL: https://jira.jboss.org/jira/browse/JBESB-2184 Project: JBoss ESB Issue Type: Bug Security Level: Public(Everyone can see) Components: Transports Affects Versions: 4.4 Environment: 4 node JBoss ESB 4.4 cluster, clustered JBM 1.4.0 SP3, replicated MySQL 5.0 Reporter: Joakim Sandström JmsCourier ends up in endless loop during cluster failover, due to a bug in JmsCourier.jmsConnectRetry. The 'failed to route' JMSException happens outside the JmsCourier.jmsConnectRetry, thus JmsCourier.jmsConnectRetry is invoked over and over again, and maxRetry is always reset in the method. final int maxRetry = 5; // TODO Magic number here!!! for (int i1 = 0; i1 < maxRetry; i1++) { 2008-11-10 16:01:29,303 ERROR [org.jboss.messaging.util.ExceptionUtil] SessionEndpoint[16-txdx5dnf-1-198q5dnf-8hclqf-kn414q1] send [le-nshn6dnf-1-198q5dnf-8hclqf-kn41 4q1] javax.jms.JMSException: Failed to route Reference[446473]:RELIABLE to myChannel at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:743) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:795) at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:101) at org.jboss.remoting.Client.invoke(Client.java:1634) at org.jboss.remoting.Client.invoke(Client.java:548) at org.jboss.remoting.Client.invoke(Client.java:536) at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189) at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160) at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$send$aop(ClientSessionDelegate.java:477) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.SessionAspect.handleSend(SessionAspect.java:632) at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect29.invoke(SessionAspect29.java) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:114) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java) at org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:278) at org.jboss.aop.advice.org.jboss.jms.client.container.ProducerAspect47.invoke(ProducerAspect47.java) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:164) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:207) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:145) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:136) at org.jboss.internal.soa.esb.couriers.JmsCourier.deliver(JmsCourier.java:252) at org.jboss.internal.soa.esb.couriers.JmsCourier.deliver(JmsCourier.java:213) at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.deliver(TwoWayCourierImpl.java:201) at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.attemptDelivery(ServiceInvoker.java:636) at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.access$2(ServiceInvoker.java:570) at org.jboss.soa.esb.client.ServiceInvoker.post(ServiceInvoker.java:371) at org.jboss.soa.esb.client.ServiceInvoker.deliverAsync(ServiceInvoker.java:252) at org.jboss.soa.esb.client.MessageMulticaster.sendToSubset(MessageMulticaster.java:144) at org.jboss.soa.esb.actions.ContentBasedWiretap.routeMessage(ContentBasedWiretap.java:187) at org.jboss.soa.esb.actions.ContentBasedWiretap.process(ContentBasedWiretap.java:153) at org.jboss.soa.esb.actions.ContentBasedRouter.process(ContentBasedRouter.java:58) at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.process(ActionProcessingPipeline.java:316) at org.jboss.soa.esb.listeners.message.MessageAwareListener$TransactionalRunner.run(MessageAwareListener.java:530) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) 2008-11-10 16:01:29,306 DEBUG [org.jboss.internal.soa.esb.couriers.JmsCourier] JMS error. Attempting JMS reconnect. javax.jms.JMSException: Failed to route Reference[446473]:RELIABLE to myChannel at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:743) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:795) at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:101) at org.jboss.remoting.Client.invoke(Client.java:1634) at org.jboss.remoting.Client.invoke(Client.java:548) at org.jboss.remoting.Client.invoke(Client.java:536) at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189) at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160) at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$send$aop(ClientSessionDelegate.java:477) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.SessionAspect.handleSend(SessionAspect.java:632) at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect29.invoke(SessionAspect29.java) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:114) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java) at org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:278) at org.jboss.aop.advice.org.jboss.jms.client.container.ProducerAspect47.invoke(ProducerAspect47.java) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:164) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:207) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:145) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:136) at org.jboss.internal.soa.esb.couriers.JmsCourier.deliver(JmsCourier.java:252) at org.jboss.internal.soa.esb.couriers.JmsCourier.deliver(JmsCourier.java:213) at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.deliver(TwoWayCourierImpl.java:201) at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.attemptDelivery(ServiceInvoker.java:636) at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.access$2(ServiceInvoker.java:570) at org.jboss.soa.esb.client.ServiceInvoker.post(ServiceInvoker.java:371) at org.jboss.soa.esb.client.ServiceInvoker.deliverAsync(ServiceInvoker.java:252) at org.jboss.soa.esb.client.MessageMulticaster.sendToSubset(MessageMulticaster.java:144) at org.jboss.soa.esb.actions.ContentBasedWiretap.routeMessage(ContentBasedWiretap.java:187) at org.jboss.soa.esb.actions.ContentBasedWiretap.process(ContentBasedWiretap.java:153) at org.jboss.soa.esb.actions.ContentBasedRouter.process(ContentBasedRouter.java:58) at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.process(ActionProcessingPipeline.java:316) at org.jboss.soa.esb.listeners.message.MessageAwareListener$TransactionalRunner.run(MessageAwareListener.java:530) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) 2008-11-10 16:01:29,335 DEBUG [org.jboss.internal.soa.esb.couriers.JmsCourier] JMSCourier deliveryMode: 2, peristent:true 2008-11-10 16:01:29,335 ERROR [org.jboss.messaging.util.ExceptionUtil] SessionEndpoint[16-txdx5dnf-1-198q5dnf-8hclqf-kn414q1] send [me-jthn6dnf-1-198q5dnf-8hclqf-kn41 4q1] javax.jms.JMSException: Failed to route Reference[446474]:RELIABLE to myChannel at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:743) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383)