Author: shawkins Date: 2012-01-16 21:12:12 -0500 (Mon, 16 Jan 2012) New Revision: 3795 Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html trunk/common-core/src/main/java/org/teiid/core/types/BlobType.java trunk/common-core/src/main/java/org/teiid/core/types/ClobType.java trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java trunk/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java trunk/common-core/src/test/java/org/teiid/core/types/TestClobValue.java trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentValueSource.java trunk/engine/src/main/java/org/teiid/query/processor/relational/GroupingNode.java trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java trunk/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java Log: TEIID-1465 TEIID-1834 adding the ability to perform a padded comparison and an option to use comparable lobs Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-17 02:12:12 UTC (rev 3795) @@ -35,7 +35,9 @@ <LI>translators may indicate support for dependent join handling </UL> <LI><B>Continuous Asynch Queries</B> to process plans in a streamed window fashion the TeiidStatement/TeiidPreparedStatement methods now take a RequestOptions object to specify continuous mode. See the Client and Developers Guides for more. - <LI><B>Texttable selectors</B> - can be used to selectively parse only record lines matching a given selector string. Selectors may also be used for column values to join data from other records positionally. + <LI><B>Texttable selectors</B> - can be used to selectively parse only record lines matching a given selector string. Selectors may also be used for column values to join data from other records positionally. + <LI><B>Comparable LOBs</B> - the system property org.teiid.comparableLobs can be set to use CLOB and BLOB values in comparison/sorting/grouping operations. + <LI><B>Padded String Comparison</B> - the system property org.teiid.padSpace can be set to effectively right pad strings to the same length for comparison. </UL> <h2><a name="Compatibility">Compatibility Issues</a></h2> Modified: trunk/common-core/src/main/java/org/teiid/core/types/BlobType.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/BlobType.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/common-core/src/main/java/org/teiid/core/types/BlobType.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -26,7 +26,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.ObjectInput; -import java.io.ObjectOutput; import java.io.OutputStream; import java.sql.Blob; import java.sql.SQLException; @@ -40,7 +39,7 @@ /** * Represent a value of type "blob", which can be streamable from client */ -public final class BlobType extends Streamable<Blob> implements Blob { +public final class BlobType extends Streamable<Blob> implements Blob, Comparable<BlobType> { private static final long serialVersionUID = 1294191629070433450L; @@ -61,8 +60,8 @@ /** * @see java.sql.Blob#getBytes(long, int) */ - public byte[] getBytes(long pos, int length) throws SQLException { - return this.reference.getBytes(pos, length); + public byte[] getBytes(long pos, int len) throws SQLException { + return this.reference.getBytes(pos, len); } @Override @@ -120,9 +119,9 @@ this.reference.free(); } - public InputStream getBinaryStream(long pos, long length) + public InputStream getBinaryStream(long pos, long len) throws SQLException { - return this.reference.getBinaryStream(pos, length); + return this.reference.getBinaryStream(pos, len); } public static SerialBlob createBlob(byte[] bytes) { @@ -171,4 +170,27 @@ } } + @Override + public int compareTo(BlobType o) { + try { + InputStream is1 = this.getBinaryStream(); + InputStream is2 = o.getBinaryStream(); + long len1 = this.length(); + long len2 = o.length(); + long n = Math.min(len1, len2); + for (long i = 0; i < n; i++) { + int b1 = is1.read(); + int b2 = is2.read(); + if (b1 != b2) { + return b1 - b2; + } + } + return Long.signum(len1 - len2); + } catch (SQLException e) { + throw new TeiidRuntimeException(e); + } catch (IOException e) { + throw new TeiidRuntimeException(e); + } + } + } Modified: trunk/common-core/src/main/java/org/teiid/core/types/ClobType.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/ClobType.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/common-core/src/main/java/org/teiid/core/types/ClobType.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -43,7 +43,7 @@ * This is wrapper on top of a "clob" object, which implements the "java.sql.Clob" * interface. This class also implements the Streamable interface */ -public final class ClobType extends Streamable<Clob> implements Clob, Sequencable { +public final class ClobType extends Streamable<Clob> implements Clob, Sequencable, Comparable<ClobType> { private static final long serialVersionUID = 2753412502127824104L; @@ -71,8 +71,8 @@ /** * @see java.sql.Clob#getSubString(long, int) */ - public String getSubString(long pos, int length) throws SQLException { - return this.reference.getSubString(pos, length); + public String getSubString(long pos, int len) throws SQLException { + return this.reference.getSubString(pos, len); } @Override @@ -200,8 +200,8 @@ this.reference.free(); } - public Reader getCharacterStream(long pos, long length) throws SQLException { - return this.reference.getCharacterStream(pos, length); + public Reader getCharacterStream(long pos, long len) throws SQLException { + return this.reference.getCharacterStream(pos, len); } public static SerialClob createClob(char[] chars) { @@ -260,5 +260,28 @@ r.close(); } } + + @Override + public int compareTo(ClobType o) { + try { + Reader cs1 = this.getCharacterStream(); + Reader cs2 = o.getCharacterStream(); + long len1 = this.length(); + long len2 = o.length(); + long n = Math.min(len1, len2); + for (long i = 0; i < n; i++) { + int c1 = cs1.read(); + int c2 = cs2.read(); + if (c1 != c2) { + return c1 - c2; + } + } + return Long.signum(len1 - len2); + } catch (SQLException e) { + throw new TeiidRuntimeException(e); + } catch (IOException e) { + throw new TeiidRuntimeException(e); + } + } } Modified: trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -66,6 +66,8 @@ public class DataTypeManager { private static final boolean USE_VALUE_CACHE = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.useValueCache", false); //$NON-NLS-1$ + private static final boolean COMPARABLE_LOBS = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.comparableLobs", false); //$NON-NLS-1$ + public static final boolean PAD_SPACE = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.padSpace", false); //$NON-NLS-1$ private static boolean valueCacheEnabled; @@ -835,8 +837,8 @@ public static boolean isNonComparable(String type) { return DataTypeManager.DefaultDataTypes.OBJECT.equals(type) - || DataTypeManager.DefaultDataTypes.BLOB.equals(type) - || DataTypeManager.DefaultDataTypes.CLOB.equals(type) + || (!COMPARABLE_LOBS && DataTypeManager.DefaultDataTypes.BLOB.equals(type)) + || (!COMPARABLE_LOBS && DataTypeManager.DefaultDataTypes.CLOB.equals(type)) || DataTypeManager.DefaultDataTypes.XML.equals(type); } @@ -873,4 +875,14 @@ } return stringCache.getValue(value); } + + public static boolean isHashable(Class<?> type) { + if (type == DataTypeManager.DefaultDataClasses.STRING) { + return !PAD_SPACE; + } + return !(type == DataTypeManager.DefaultDataClasses.BIG_DECIMAL + || type == DataTypeManager.DefaultDataClasses.BLOB + || type == DataTypeManager.DefaultDataClasses.CLOB); + } + } Modified: trunk/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java =================================================================== --- trunk/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -71,4 +71,14 @@ assertEquals(testString, new String(read.getBytes(1, (int)blob.length()))); } + public void testBlobCompare() throws Exception { + String testString = "this is test clob"; //$NON-NLS-1$ + SerialBlob blob = new SerialBlob(testString.getBytes()); + BlobType bv = new BlobType(blob); + + SerialBlob blob1 = new SerialBlob(testString.getBytes()); + BlobType bv1 = new BlobType(blob1); + assertEquals(0, bv1.compareTo(bv)); + } + } Modified: trunk/common-core/src/test/java/org/teiid/core/types/TestClobValue.java =================================================================== --- trunk/common-core/src/test/java/org/teiid/core/types/TestClobValue.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/common-core/src/test/java/org/teiid/core/types/TestClobValue.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -127,4 +127,14 @@ assertEquals("aa", clob.getSubString(1, 3)); } + public void testClobCompare() throws Exception { + String testString = "this is test clob"; //$NON-NLS-1$ + SerialClob clob = new SerialClob(testString.toCharArray()); + ClobType ct = new ClobType(clob); + + SerialClob clob1 = new SerialClob(testString.toCharArray()); + ClobType ct1 = new ClobType(clob1); + assertEquals(0, ct1.compareTo(ct)); + } + } Modified: trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml =================================================================== --- trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml 2012-01-17 02:12:12 UTC (rev 3795) @@ -45,5 +45,15 @@ Set to true to parse exact fixed point literals, e.g. 1.0, as double values rather than as decimal/BigDecimal values and to return a double value from the AVG function for integral values in the same way as releases earlier than 8.0. </para> </listitem> + <listitem> + <para><emphasis>org.teiid.comparableLobs</emphasis> - defaults to false. + Set to true to allow blob and clob values to be comparable in Teiid. Source type metadata will determine if the comparison can be pushed down. + </para> + </listitem> + <listitem> + <para><emphasis>org.teiid.padSpace</emphasis> - defaults to false. + Set to true to compare strings as if PAD SPACE collation is being used, that is strings are effectively right padded to the same length for comparison. If this property is set, it is not necessary to use the trimStrings translator option. + </para> + </listitem> </itemizedlist> </appendix> \ No newline at end of file Modified: trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -324,7 +324,7 @@ if (leftValue == rightValue) { return 0; } - return ((Comparable<Object>)leftValue).compareTo(rightValue); + return Constant.compare((Comparable<?>)leftValue, (Comparable<?>)rightValue); } public Boolean evaluate(MatchCriteria criteria, List<?> tuple) Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentValueSource.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentValueSource.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentValueSource.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -87,7 +87,8 @@ index = buffer.getSchema().indexOf(valueExpression); } Assertion.assertTrue(index != -1); - if (((Expression)buffer.getSchema().get(index)).getType() == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) { + Class<?> type = ((Expression)buffer.getSchema().get(index)).getType(); + if (!DataTypeManager.isHashable(type)) { result = new TreeSet<Object>(); } else { result = new HashSet<Object>(); Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/GroupingNode.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/GroupingNode.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/GroupingNode.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -403,25 +403,7 @@ if (indexes == null) { return true; } - for(int i=indexes.length-1; i>=0; i--) { - Object oldValue = oldTuple.get(indexes[i]); - Object newValue = newTuple.get(indexes[i]); - - if(oldValue == null) { - if(newValue != null) { - return false; - } - } else { - if(newValue == null) { - return false; - } - if(! oldValue.equals(newValue)) { - return false; - } - } - } - - return true; + return MergeJoinStrategy.compareTuples(newTuple, oldTuple, indexes, indexes, true) == 0; } private void updateAggregates(List<?> tuple) Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -25,6 +25,7 @@ import java.util.List; import org.teiid.language.SortSpecification.NullOrdering; +import org.teiid.query.sql.symbol.Constant; /** * This class can be used for comparing lists of elements, when the fields to @@ -173,7 +174,7 @@ return -1; } } else { - compare = param1.compareTo(param2); + compare = Constant.compare(param1, param2); } if (compare != 0) { boolean asc = orderTypes != null?orderTypes.get(k):this.ascendingOrder; Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -28,6 +28,7 @@ import org.teiid.core.TeiidProcessingException; import org.teiid.query.processor.relational.SourceState.ImplicitBuffer; import org.teiid.query.sql.lang.JoinType; +import org.teiid.query.sql.symbol.Constant; /** @@ -302,11 +303,17 @@ List rightProbe, int[] leftExpressionIndecies, int[] rightExpressionIndecies) { - for (int i = 0; i < leftExpressionIndecies.length; i++) { + return compareTuples(leftProbe, rightProbe, leftExpressionIndecies, + rightExpressionIndecies, grouping); + } + + public static int compareTuples(List<?> leftProbe, List<?> rightProbe, + int[] leftExpressionIndecies, int[] rightExpressionIndecies, boolean nullEquals) { + for (int i = 0; i < leftExpressionIndecies.length; i++) { Object leftValue = leftProbe.get(leftExpressionIndecies[i]); Object rightValue = rightProbe.get(rightExpressionIndecies[i]); if (rightValue == null) { - if (grouping && leftValue == null) { + if (nullEquals && leftValue == null) { continue; } return -1; @@ -316,7 +323,7 @@ return 1; } - int c = ((Comparable)rightValue).compareTo(leftValue); + int c = Constant.compare((Comparable)rightValue, (Comparable)leftValue); if (c != 0) { return c; } Modified: trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -25,6 +25,7 @@ import java.math.BigDecimal; import java.util.List; +import org.teiid.core.types.ClobType; import org.teiid.core.types.DataTypeManager; import org.teiid.query.QueryPlugin; import org.teiid.query.sql.LanguageVisitor; @@ -223,9 +224,60 @@ if (o.isNull()) { return 1; } - return ((Comparable)this.value).compareTo(o.getValue()); + return compare((Comparable<?>)this.value, (Comparable<?>)o.getValue()); } + /** + * Compare the given non-null values + * @param o1 + * @param o2 + * @return + */ + public final static int compare(Comparable o1, Comparable o2) { + if (DataTypeManager.PAD_SPACE) { + if (o1 instanceof String) { + CharSequence s1 = (CharSequence)o1; + CharSequence s2 = (CharSequence)o2; + return comparePadded(s1, s2); + } else if (o1 instanceof ClobType) { + CharSequence s1 = ((ClobType)o1).getCharSequence(); + CharSequence s2 = ((ClobType)o2).getCharSequence(); + return comparePadded(s1, s2); + } + } + return o1.compareTo(o2); + } + + final static int comparePadded(CharSequence s1, CharSequence s2) { + int len1 = s1.length(); + int len2 = s2.length(); + int n = Math.min(len1, len2); + int i = 0; + int result = 0; + for (; i < n; i++) { + char c1 = s1.charAt(i); + char c2 = s2.charAt(i); + if (c1 != c2) { + result = c1 - c2; + break; + } + } + if (result == 0 && len1 != len2) { + result = len1 - len2; + } + for (int j = i; j < len1; j++) { + if (s1.charAt(j) != ' ') { + return result; + } + } + for (int j = i; j < len2; j++) { + if (s2.charAt(j) != ' ') { + return result; + } + } + return 0; + } + public boolean isBindEligible() { return bindEligible; } Modified: trunk/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java =================================================================== --- trunk/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -69,7 +69,7 @@ LobManager lobManager = new LobManager(new int[] {0, 1}, fs); lobManager.setMaxMemoryBytes(4); - List<Streamable<? extends Object>> tuple = Arrays.asList(clob, blob); + List<?> tuple = Arrays.asList(clob, blob); lobManager.updateReferences(tuple, ReferenceMode.CREATE); lobManager.persist(); Modified: trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java 2012-01-16 17:31:15 UTC (rev 3794) +++ trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java 2012-01-17 02:12:12 UTC (rev 3795) @@ -22,20 +22,15 @@ package org.teiid.query.sql.symbol; -import junit.framework.TestCase; +import static org.junit.Assert.*; +import org.junit.Test; import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.UnitTestUtil; -public class TestConstant extends TestCase { +public class TestConstant { - // ################################## FRAMEWORK ################################ - - public TestConstant(String name) { - super(name); - } - // ################################## TEST HELPERS ################################ public static final Constant sample1() { @@ -50,7 +45,7 @@ // ################################## ACTUAL TESTS ################################ - public void testString() { + @Test public void testString() { String s = "the string"; //$NON-NLS-1$ Constant c = new Constant(s); assertEquals("Value is incorrect: ", s, c.getValue()); //$NON-NLS-1$ @@ -65,7 +60,7 @@ assertEquals("Cloned object not equal to original: ", c, cc); //$NON-NLS-1$ } - public void testInteger() { + @Test public void testInteger() { Integer i = new Integer(5); Constant c = new Constant(i); assertEquals("Value is incorrect: ", i, c.getValue()); //$NON-NLS-1$ @@ -80,7 +75,7 @@ assertEquals("Cloned object not equal to original: ", c, cc); //$NON-NLS-1$ } - public void testNoTypeNull() { + @Test public void testNoTypeNull() { Constant c = new Constant(null); assertEquals("Value is incorrect: ", null, c.getValue()); //$NON-NLS-1$ assertEquals("Type is incorrect: ", DataTypeManager.DefaultDataClasses.NULL, c.getType()); //$NON-NLS-1$ @@ -94,7 +89,7 @@ assertEquals("Cloned object not equal to original: ", c, cc); //$NON-NLS-1$ } - public void testTypedNull() { + @Test public void testTypedNull() { Constant c = new Constant(null, DataTypeManager.DefaultDataClasses.STRING); assertEquals("Value is incorrect: ", null, c.getValue()); //$NON-NLS-1$ assertEquals("Type is incorrect: ", DataTypeManager.DefaultDataClasses.STRING, c.getType()); //$NON-NLS-1$ @@ -111,7 +106,7 @@ assertEquals("Typed null not equal to non-typed null: ", c, c3); //$NON-NLS-1$ } - public void testClone() { + @Test public void testClone() { // Use this object as the "object"-type value for c1 StringBuffer value = new StringBuffer("x"); //$NON-NLS-1$ @@ -129,20 +124,20 @@ assertTrue("Cloned object has not changed, but should have", ((StringBuffer)copy.getValue()).toString().equals("xy")); //$NON-NLS-1$ //$NON-NLS-2$ } - public void testSelfEquivalence(){ + @Test public void testSelfEquivalence(){ Object s1 = sample1(); int equals = 0; UnitTestUtil.helpTestEquivalence(equals, s1, s1); } - public void testEquivalence(){ + @Test public void testEquivalence(){ Object s1 = sample1(); Object s1a = sample1(); int equals = 0; UnitTestUtil.helpTestEquivalence(equals, s1, s1a); } - public void testNonEquivalence(){ + @Test public void testNonEquivalence(){ Object s1 = sample1(); Object s2 = sample2(); int equals = -1; @@ -153,5 +148,14 @@ // this exception should be thrown } } + + @Test public void testPaddedStringComparison(){ + assertEquals(1, Constant.comparePadded("a", "")); + assertEquals(0, Constant.comparePadded("a", "a ")); + assertEquals(-24, Constant.comparePadded("ab ", "az ")); + assertEquals(66, Constant.comparePadded("ab ", "a ")); + assertEquals(0, Constant.comparePadded("a1 ", "a1")); + + } }