[ https://issues.jboss.org/browse/TEIID-5185?page=com.atlassian.jira.plugin... ] Krum Bakalsky updated TEIID-5185: --------------------------------- Description: We have a server-side application that is powered by Teiid version 9.3.4, and that exposes a Teiid JDBC endpoint to its clients. A couple of app instances are configured to sit behind a VIP load balancer, that works on the TCP level. We have been experiencing intermittently following problem, when we run Teiid 9.3.4 client to execute multi-batch queries to the server: * client successfully fetches few batches from server; * client hangs and its thread blocks on waiting for the next batch to be delivered on the network; * after 1 hour client throws exceptions that Socket is closed. Client thread's call stack: {code} "Thread Group 1-1" #41 prio=6 os_prio=0 tid=0x000000005e03a800 nid=0x17bc runnable [0x00000000623ae000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.io.BufferedInputStream.fill(Unknown Source) at java.io.BufferedInputStream.read(Unknown Source) - locked <0x00000000f5f32420> (a org.teiid.core.util.AccessibleBufferedInputStream) at java.io.DataInputStream.readInt(Unknown Source) at org.teiid.netty.handler.codec.serialization.ObjectDecoderInputStream.readObjectOverride(ObjectDecoderInputStream.java:97) at java.io.ObjectInputStream.readObject(Unknown Source) at org.teiid.net.socket.OioOjbectChannelFactory$OioObjectChannel.read(OioOjbectChannelFactory.java:117) at org.teiid.net.socket.SocketServerInstanceImpl.read(SocketServerInstanceImpl.java:324) at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at org.teiid.net.socket.SocketServerConnectionFactory$ShutdownHandler.invoke(SocketServerConnectionFactory.java:98) at com.sun.proxy.$Proxy21.read(Unknown Source) at org.teiid.net.socket.SocketServerInstanceImpl$RemoteInvocationHandler$1.get(SocketServerInstanceImpl.java:427) at org.teiid.jdbc.ResultSetImpl.getResults(ResultSetImpl.java:449) at org.teiid.jdbc.ResultSetImpl.requestBatch(ResultSetImpl.java:382) at org.teiid.jdbc.BatchResults.requestBatchAndWait(BatchResults.java:227) at org.teiid.jdbc.BatchResults.requestNextBatch(BatchResults.java:142) at org.teiid.jdbc.BatchResults.hasNext(BatchResults.java:253) at org.teiid.jdbc.ResultSetImpl.hasNext(ResultSetImpl.java:510) at org.teiid.jdbc.ResultSetImpl.next(ResultSetImpl.java:259) at org.apache.commons.dbcp2.DelegatingResultSet.next(DelegatingResultSet.java:191) at org.apache.commons.dbcp2.DelegatingResultSet.next(DelegatingResultSet.java:191) at org.apache.jmeter.protocol.jdbc.AbstractJDBCTestElement.getStringFromResultSet(AbstractJDBCTestElement.java:535) at org.apache.jmeter.protocol.jdbc.AbstractJDBCTestElement.execute(AbstractJDBCTestElement.java:174) at org.apache.jmeter.protocol.jdbc.sampler.JDBCSampler.sample(JDBCSampler.java:84) at org.apache.jmeter.threads.JMeterThread.executeSamplePackage(JMeterThread.java:491) at org.apache.jmeter.threads.JMeterThread.processSampler(JMeterThread.java:425) at org.apache.jmeter.threads.JMeterThread.run(JMeterThread.java:254) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None {code} Client side exception after 1 hour: {code} 2017-12-19 13:17:07,160 WARN o.a.j.p.j.AbstractJDBCTestElement: Error closing ResultSet org.teiid.jdbc.TeiidSQLException: TEIID20013 java.net.SocketException: Socket closed at org.teiid.jdbc.TeiidSQLException.create(TeiidSQLException.java:135) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.jdbc.TeiidSQLException.create(TeiidSQLException.java:71) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.jdbc.ResultSetImpl.close(ResultSetImpl.java:168) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.apache.commons.dbcp2.DelegatingResultSet.close(DelegatingResultSet.java:170) ~[commons-dbcp2-2.1.1.jar:2.1.1] at org.apache.commons.dbcp2.DelegatingResultSet.close(DelegatingResultSet.java:170) ~[commons-dbcp2-2.1.1.jar:2.1.1] at org.apache.jmeter.protocol.jdbc.AbstractJDBCTestElement.close(AbstractJDBCTestElement.java:607) [ApacheJMeter_jdbc.jar:3.2 r1790748] at org.apache.jmeter.protocol.jdbc.AbstractJDBCTestElement.execute(AbstractJDBCTestElement.java:176) [ApacheJMeter_jdbc.jar:3.2 r1790748] at org.apache.jmeter.protocol.jdbc.sampler.JDBCSampler.sample(JDBCSampler.java:84) [ApacheJMeter_jdbc.jar:3.2 r1790748] at org.apache.jmeter.threads.JMeterThread.executeSamplePackage(JMeterThread.java:491) [ApacheJMeter_core.jar:3.2 r1790748] at org.apache.jmeter.threads.JMeterThread.processSampler(JMeterThread.java:425) [ApacheJMeter_core.jar:3.2 r1790748] at org.apache.jmeter.threads.JMeterThread.run(JMeterThread.java:254) [ApacheJMeter_core.jar:3.2 r1790748] at java.lang.Thread.run(Unknown Source) [?:1.8.0_121] Caused by: org.teiid.core.TeiidComponentException: TEIID20013 java.net.SocketException: Socket closed at org.teiid.client.util.ExceptionUtil.convertException(ExceptionUtil.java:65) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerInstanceImpl$RemoteInvocationHandler.invoke(SocketServerInstanceImpl.java:445) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerConnection$1.invoke(SocketServerConnection.java:293) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy24.closeRequest(Unknown Source) ~[?:?] at org.teiid.jdbc.ResultSetImpl.close(ResultSetImpl.java:164) ~[teiid-9.3.4-jdbc.jar:9.3.4] ... 9 more Caused by: org.teiid.net.socket.SingleInstanceCommunicationException: TEIID20013 java.net.SocketException: Socket closed at org.teiid.net.socket.SocketServerInstanceImpl.send(SocketServerInstanceImpl.java:217) ~[teiid-9.3.4-jdbc.jar:9.3.4] at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:1.8.0_121] at java.lang.reflect.Method.invoke(Unknown Source) ~[?:1.8.0_121] at org.teiid.net.socket.SocketServerConnectionFactory$ShutdownHandler.invoke(SocketServerConnectionFactory.java:98) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy21.send(Unknown Source) ~[?:?] at org.teiid.net.socket.SocketServerInstanceImpl$RemoteInvocationHandler.invoke(SocketServerInstanceImpl.java:433) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerConnection$1.invoke(SocketServerConnection.java:293) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy24.closeRequest(Unknown Source) ~[?:?] at org.teiid.jdbc.ResultSetImpl.close(ResultSetImpl.java:164) ~[teiid-9.3.4-jdbc.jar:9.3.4] ... 9 more Caused by: java.util.concurrent.ExecutionException: java.net.SocketException: Socket closed at org.teiid.client.util.ResultsFuture.convertResult(ResultsFuture.java:104) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.client.util.ResultsFuture.get(ResultsFuture.java:99) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerInstanceImpl.send(SocketServerInstanceImpl.java:214) ~[teiid-9.3.4-jdbc.jar:9.3.4] at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:1.8.0_121] at java.lang.reflect.Method.invoke(Unknown Source) ~[?:1.8.0_121] at org.teiid.net.socket.SocketServerConnectionFactory$ShutdownHandler.invoke(SocketServerConnectionFactory.java:98) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy21.send(Unknown Source) ~[?:?] at org.teiid.net.socket.SocketServerInstanceImpl$RemoteInvocationHandler.invoke(SocketServerInstanceImpl.java:433) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerConnection$1.invoke(SocketServerConnection.java:293) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy24.closeRequest(Unknown Source) ~[?:?] at org.teiid.jdbc.ResultSetImpl.close(ResultSetImpl.java:164) ~[teiid-9.3.4-jdbc.jar:9.3.4] ... 9 more Caused by: java.net.SocketException: Socket closed at java.net.SocketOutputStream.socketWrite(Unknown Source) ~[?:1.8.0_121] at java.net.SocketOutputStream.write(Unknown Source) ~[?:1.8.0_121] at java.io.DataOutputStream.write(Unknown Source) ~[?:1.8.0_121] at org.teiid.core.util.MultiArrayOutputStream.writeTo(MultiArrayOutputStream.java:94) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.netty.handler.codec.serialization.ObjectEncoderOutputStream.writeObjectOverride(ObjectEncoderOutputStream.java:70) ~[teiid-9.3.4-jdbc.jar:9.3.4] at java.io.ObjectOutputStream.writeObject(Unknown Source) ~[?:1.8.0_121] at org.teiid.net.socket.OioOjbectChannelFactory$OioObjectChannel.write(OioOjbectChannelFactory.java:136) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerInstanceImpl.send(SocketServerInstanceImpl.java:213) ~[teiid-9.3.4-jdbc.jar:9.3.4] at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:1.8.0_121] at java.lang.reflect.Method.invoke(Unknown Source) ~[?:1.8.0_121] at org.teiid.net.socket.SocketServerConnectionFactory$ShutdownHandler.invoke(SocketServerConnectionFactory.java:98) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy21.send(Unknown Source) ~[?:?] at org.teiid.net.socket.SocketServerInstanceImpl$RemoteInvocationHandler.invoke(SocketServerInstanceImpl.java:433) ~[teiid-9.3.4-jdbc.jar:9.3.4] at org.teiid.net.socket.SocketServerConnection$1.invoke(SocketServerConnection.java:293) ~[teiid-9.3.4-jdbc.jar:9.3.4] at com.sun.proxy.$Proxy24.closeRequest(Unknown Source) ~[?:?] at org.teiid.jdbc.ResultSetImpl.close(ResultSetImpl.java:164) ~[teiid-9.3.4-jdbc.jar:9.3.4] ... 9 more {code} was: We have a server-side application that is powered by Teiid version 9.3.4, and that exposes a Teiid JDBC endpoint to its clients. A couple of app instances are configured to sit behind a VIP load balancer, that works on the TCP level. We have been experiencing intermittently following problem, when we run Teiid 9.3.4 client to execute multi-batch queries to the server: * client successfully fetches few batches from server; * client hangs and its thread blocks on waiting for the next batch to be delivered on the network; * after 1 hour client throws exceptions that Socket is closed. -- This message was sent by Atlassian JIRA (v7.5.0#75005)