[teiid-commits] teiid SVN: r4396 - in trunk: client/src/main/java/org/teiid and 5 other directories.

Author: shawkins Date: 2012-09-04 08:26:59 -0400 (Tue, 04 Sep 2012) New Revision: 4396 Removed: trunk/client/src/main/java/org/teiid/adminapi/ Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html trunk/client/src/main/java/org/teiid/client/BatchSerializer.java trunk/client/src/main/java/org/teiid/jdbc/JDBCPlugin.java trunk/client/src/main/resources/org/teiid/jdbc/i18n.properties trunk/client/src/test/java/org/teiid/client/TestBatchSerializer.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java Log: TEIID-2175 changing the date/time serialization Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-09-04 11:53:12 UTC (rev 4395) +++ trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-09-04 12:26:59 UTC (rev 4396) @@ -42,9 +42,11 @@ <li>decodeinteger/decodestring have been deprecated. A CASE expression should be used instead. </ul> -<h4>from 8.0</h4> +<h4>from 8.1</h4> <ul> <li>TEIID-2166 array_get will return null if the index is out of bounds rather than raising an error. + <li>TEIID-2175 for 8.0 and 8.1 clients the server will check if serialized date/time values fall outside of 32-bit value ranges (year 1900 - 9999 for dates and times between years 1901 and 2038) and throw an exception. The previous behavior was + to truncate. The exception and the use of 32 bit serialization can be avoided by setting the system property org.teiid.longDatesTimes to true. </ul> <h4>from 8.0</h4> Modified: trunk/client/src/main/java/org/teiid/client/BatchSerializer.java =================================================================== --- trunk/client/src/main/java/org/teiid/client/BatchSerializer.java 2012-09-04 11:53:12 UTC (rev 4395) +++ trunk/client/src/main/java/org/teiid/client/BatchSerializer.java 2012-09-04 12:26:59 UTC (rev 4396) @@ -54,10 +54,20 @@ /** * @since 4.2 + * + * <ul> + * <li>version 0: starts with 7.1 and uses simple serialization too broadly + * <li>version 1: starts with 8.0 uses better string, blob, clob, xml, etc. + * add varbinary support. + * however was possibly silently truncating date/time values that were + * outside of jdbc allowed values + * <li>version 2: starts with 8.2 and adds better array serialization and + * uses a safer date/time serialization + * </ul> */ public class BatchSerializer { - private static final byte CURRENT_VERSION = (byte)2; + static final byte CURRENT_VERSION = (byte)2; private BatchSerializer() {} // Uninstantiable @@ -70,13 +80,13 @@ serializers.put(DataTypeManager.DefaultDataTypes.BOOLEAN, new ColumnSerializer[] {new BooleanColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.BYTE, new ColumnSerializer[] {new ByteColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.CHAR, new ColumnSerializer[] {new CharColumnSerializer()}); - serializers.put(DataTypeManager.DefaultDataTypes.DATE, new ColumnSerializer[] {new DateColumnSerializer(), new DateColumnSerializer1()}); + serializers.put(DataTypeManager.DefaultDataTypes.DATE, new ColumnSerializer[] {new DateColumnSerializer(), new DateColumnSerializer1(), new DateColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.DOUBLE, new ColumnSerializer[] {new DoubleColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.FLOAT, new ColumnSerializer[] {new FloatColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.INTEGER, new ColumnSerializer[] {new IntColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.LONG, new ColumnSerializer[] {new LongColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.SHORT, new ColumnSerializer[] {new ShortColumnSerializer()}); - serializers.put(DataTypeManager.DefaultDataTypes.TIME, new ColumnSerializer[] {new TimeColumnSerializer(), new TimeColumnSerializer1()}); + serializers.put(DataTypeManager.DefaultDataTypes.TIME, new ColumnSerializer[] {new TimeColumnSerializer(), new TimeColumnSerializer1(), new TimeColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.TIMESTAMP, new ColumnSerializer[] {new TimestampColumnSerializer()}); serializers.put(DataTypeManager.DefaultDataTypes.STRING, new ColumnSerializer[] {defaultSerializer, new StringColumnSerializer1()}); serializers.put(DataTypeManager.DefaultDataTypes.CLOB, new ColumnSerializer[] {defaultSerializer, new ClobColumnSerializer1()}); @@ -617,18 +627,30 @@ } static int DATE_NORMALIZER = 0; + public final static long MIN_DATE_32; + public final static long MAX_DATE_32; + public final static long MIN_TIME_32; + public final static long MAX_TIME_32; static { Calendar c = Calendar.getInstance(); c.setTimeZone(TimeZone.getTimeZone("GMT")); //$NON-NLS-1$ c.set(1900, 0, 1, 0, 0, 0); c.set(Calendar.MILLISECOND, 0); - DATE_NORMALIZER = -(int)(c.getTime().getTime()/60000); //support a 32 bit range starting at this value + MIN_DATE_32 = c.getTimeInMillis(); + MAX_DATE_32 = MIN_DATE_32 + ((1l<<32)-1)*60000; + DATE_NORMALIZER = -(int)(MIN_DATE_32/60000); //support a 32 bit range starting at this value + MAX_TIME_32 = Integer.MAX_VALUE*1000l; + MIN_TIME_32 = Integer.MIN_VALUE*1000l; } private static class DateColumnSerializer1 extends ColumnSerializer { protected void writeObject(ObjectOutput out, Object obj) throws IOException { - out.writeInt((int)(((java.sql.Date)obj).getTime()/60000) + DATE_NORMALIZER); + long time = ((java.sql.Date)obj).getTime(); + if (time < MIN_DATE_32 || time > MAX_DATE_32) { + throw new IOException(JDBCPlugin.Util.gs(JDBCPlugin.Event.TEIID20029, obj.getClass().getName())); + } + out.writeInt((int)(time/60000) + DATE_NORMALIZER); } protected Object readObject(ObjectInput in) throws IOException { return new java.sql.Date(((in.readInt()&0xffffffffL) - DATE_NORMALIZER)*60000); @@ -637,7 +659,11 @@ private static class TimeColumnSerializer1 extends ColumnSerializer { protected void writeObject(ObjectOutput out, Object obj) throws IOException { - out.writeInt((int)(((Time)obj).getTime()/1000)); + long time = ((Time)obj).getTime(); + if (time < MIN_TIME_32 || time > MAX_TIME_32) { + throw new IOException(JDBCPlugin.Util.gs(JDBCPlugin.Event.TEIID20029, obj.getClass().getName())); + } + out.writeInt((int)(time/1000)); } protected Object readObject(ObjectInput in) throws IOException { return new Time((in.readInt()&0xffffffffL)*1000); Modified: trunk/client/src/main/java/org/teiid/jdbc/JDBCPlugin.java =================================================================== --- trunk/client/src/main/java/org/teiid/jdbc/JDBCPlugin.java 2012-09-04 11:53:12 UTC (rev 4395) +++ trunk/client/src/main/java/org/teiid/jdbc/JDBCPlugin.java 2012-09-04 12:26:59 UTC (rev 4396) @@ -57,6 +57,7 @@ TEIID20021, TEIID20023, TEIID20027, - TEIID20028 + TEIID20028, + TEIID20029 } } Modified: trunk/client/src/main/resources/org/teiid/jdbc/i18n.properties =================================================================== --- trunk/client/src/main/resources/org/teiid/jdbc/i18n.properties 2012-09-04 11:53:12 UTC (rev 4395) +++ trunk/client/src/main/resources/org/teiid/jdbc/i18n.properties 2012-09-04 12:26:59 UTC (rev 4396) @@ -172,4 +172,5 @@ TEIID20028=data length too big: {0} > max of {1}. You may need to adjust the maxObjectSize client setting. unexpected_element=Unexpected Element {0} encountered, expecting one of {1} ambigious_gss_selection=Either configure "java.security.krb5.conf" property or combination of "java.security.krb5.realm" and "java.security.krb5.kdc" properties. Not both. -no_gss_selection=No KRB5 configuration found. Either configure "java.security.krb5.conf" property or combination of "java.security.krb5.realm" and "java.security.krb5.kdc" properties. \ No newline at end of file +no_gss_selection=No KRB5 configuration found. Either configure "java.security.krb5.conf" property or combination of "java.security.krb5.realm" and "java.security.krb5.kdc" properties. +TEIID20029={0} value outside of 32-bit value range. Please set the system property org.teiid.longDatesTimes to true to avoid this error. \ No newline at end of file Modified: trunk/client/src/test/java/org/teiid/client/TestBatchSerializer.java =================================================================== --- trunk/client/src/test/java/org/teiid/client/TestBatchSerializer.java 2012-09-04 11:53:12 UTC (rev 4395) +++ trunk/client/src/test/java/org/teiid/client/TestBatchSerializer.java 2012-09-04 12:26:59 UTC (rev 4396) @@ -38,6 +38,7 @@ import org.teiid.core.types.BinaryType; import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.TimestampWithTimezone; +import org.teiid.query.unittest.TimestampUtil; /** @@ -45,11 +46,12 @@ */ public class TestBatchSerializer { - private static void helpTestSerialization(String[] types, List<?>[] batch) throws IOException, ClassNotFoundException { + private static void helpTestSerialization(String[] types, List<?>[] batch, byte version) throws IOException, ClassNotFoundException { ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(byteStream); List<List<?>> batchList = Arrays.asList(batch); - BatchSerializer.writeBatch(out, types, batchList); + + BatchSerializer.writeBatch(out, types, batchList, version); out.flush(); byte[] bytes = byteStream.toByteArray(); @@ -119,28 +121,33 @@ @Test public void testSerializeBasicTypes() throws Exception { // The number 8 is important here because boolean isNull information is packed into bytes, // so we want to make sure the boundary cases are handled correctly - helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(1)); // Less than 8 rows - helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(8)); // Exactly 8 rows - helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(17)); // More than 8 rows, but not a multiple of 8 - helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(120)); // A multiple of 8 rows - helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(833)); // A bunch of rows. This should also test large strings - helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(4096)); // A bunch of rows. This should also test large strings + helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(1), BatchSerializer.CURRENT_VERSION); // Less than 8 rows + helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(8), BatchSerializer.CURRENT_VERSION); // Exactly 8 rows + helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(17), BatchSerializer.CURRENT_VERSION); // More than 8 rows, but not a multiple of 8 + helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(120), BatchSerializer.CURRENT_VERSION); // A multiple of 8 rows + helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(833), BatchSerializer.CURRENT_VERSION); // A bunch of rows. This should also test large strings + helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(4096), BatchSerializer.CURRENT_VERSION); // A bunch of rows. This should also test large strings } @Test public void testSerializeLargeStrings() throws Exception { List<?> row = Arrays.asList(new Object[] {sampleString(66666)}); - helpTestSerialization(new String[] {DataTypeManager.DefaultDataTypes.STRING}, new List[] {row}); + helpTestSerialization(new String[] {DataTypeManager.DefaultDataTypes.STRING}, new List[] {row}, BatchSerializer.CURRENT_VERSION); } @Test public void testSerializeNoData() throws Exception { - helpTestSerialization(sampleBatchTypes, new List[0]); + helpTestSerialization(sampleBatchTypes, new List[0], BatchSerializer.CURRENT_VERSION); } @Test public void testSerializeDatatypeMismatch() throws Exception { try { - helpTestSerialization(new String[] {DataTypeManager.DefaultDataTypes.DOUBLE}, new List[] {Arrays.asList(new Object[] {"Hello!"})}); //$NON-NLS-1$ + helpTestSerialization(new String[] {DataTypeManager.DefaultDataTypes.DOUBLE}, new List[] {Arrays.asList(new Object[] {"Hello!"})}, BatchSerializer.CURRENT_VERSION); //$NON-NLS-1$ } catch (RuntimeException e) { assertEquals("TEIID20001 The modeled datatype double for column 0 doesn't match the runtime type \"java.lang.String\". Please ensure that the column's modeled datatype matches the expected data.", e.getMessage()); //$NON-NLS-1$ } } + + @Test(expected=IOException.class) public void testOutOfRangeDate() throws Exception { + helpTestSerialization(new String[] {DataTypeManager.DefaultDataTypes.DATE}, new List[] {Arrays.asList(TimestampUtil.createDate(-2, 0, 1))}, (byte)1); + } + } Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java 2012-09-04 11:53:12 UTC (rev 4395) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java 2012-09-04 12:26:59 UTC (rev 4396) @@ -42,6 +42,7 @@ import org.teiid.adminapi.impl.SessionMetadata; import org.teiid.adminapi.impl.VDBMetaData; import org.teiid.client.security.SessionToken; +import org.teiid.core.util.PropertiesUtils; import org.teiid.dqp.message.RequestID; import org.teiid.security.SecurityHelper; @@ -50,14 +51,13 @@ private static final long serialVersionUID = -6389893410233192977L; + private static final boolean longDatesTimes = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.longDatesTimes", false); //$NON-NLS-1$ + public enum Version { SEVEN_1("7.1", (byte)0), //$NON-NLS-1$ - SEVEN_2("7.2", (byte)0), //$NON-NLS-1$ SEVEN_3("7.3", (byte)0), //$NON-NLS-1$ SEVEN_4("7.4", (byte)0), //$NON-NLS-1$ - SEVEN_5("7.5", (byte)0), //$NON-NLS-1$ - SEVEN_6("7.6", (byte)0), //$NON-NLS-1$ - EIGHT_0("8.0", (byte)1), //$NON-NLS-1$ + EIGHT_0("8.0", (byte)(longDatesTimes?0:1)), //$NON-NLS-1$ EIGHT_2("8.2", (byte)2); //$NON-NLS-1$ private String string;