Author: shawkins Date: 2012-01-30 21:07:37 -0500 (Mon, 30 Jan 2012) New Revision: 3835 Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html trunk/engine/src/main/java/org/teiid/query/QueryPlugin.java trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java trunk/engine/src/main/resources/org/teiid/query/i18n.properties trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java Log: TEIID-1916 adding collation support via a locale system property. Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-31 02:07:37 UTC (rev 3835) @@ -36,8 +36,12 @@ </UL> <LI><B>Continuous Asynch Queries</B> to process plans in a streamed window fashion the TeiidStatement/TeiidPreparedStatement methods now take a RequestOptions object to specify continuous mode. See the Client and Developers Guides for more. <LI><B>Texttable selectors</B> - can be used to selectively parse only record lines matching a given selector string. Selectors may also be used for column values to join data from other records positionally. - <LI><B>Comparable LOBs</B> - the system property org.teiid.comparableLobs can be set to use CLOB and BLOB values in comparison/sorting/grouping operations. - <LI><B>Padded String Comparison</B> - the system property org.teiid.padSpace can be set to effectively right pad strings to the same length for comparison. + <LI><B>Enhanced Comparison Support</B> - see the Admin Guide more. + <UL> + <LI><B>Comparable LOBs</B> - the system property org.teiid.comparableLobs can be set to use CLOB and BLOB values in comparison/sorting/grouping operations. + <LI><B>Padded Comparison</B> - the system property org.teiid.padSpace can be set to effectively right pad strings to the same length for comparison. + <LI><B>Collation Support</B> - the system property org.teiid.collationLocale can be set to use a different collation than the Java UTF-16 default. + </UL> <LI><B>VARBINARY type support</B> - the Teiid VARBINARY type can now be used to support source BINARY and VARBINARY types. </UL> Modified: trunk/engine/src/main/java/org/teiid/query/QueryPlugin.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/QueryPlugin.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/QueryPlugin.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -75,5 +75,7 @@ TEIID30029, // unexpected format TEIID30030, // unexpected exp1 TEIID30031, // unexpected exp2 + TEIID30032, // invalid collation locale + TEIID30033, // using collation locale } } Modified: trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -298,35 +298,6 @@ return compare(criteria, leftValue, rightValue); } - public static Boolean compare(CompareCriteria criteria, Object leftValue, - Object rightValue) throws ExpressionEvaluationException { - switch(criteria.getOperator()) { - case CompareCriteria.EQ: - return Boolean.valueOf(compareValues(leftValue, rightValue) == 0); - case CompareCriteria.NE: - return Boolean.valueOf(compareValues(leftValue, rightValue) != 0); - case CompareCriteria.LT: - return Boolean.valueOf((compareValues(leftValue, rightValue) < 0)); - case CompareCriteria.LE: - return Boolean.valueOf((compareValues(leftValue, rightValue) <= 0)); - case CompareCriteria.GT: - return Boolean.valueOf((compareValues(leftValue, rightValue) > 0)); - case CompareCriteria.GE: - return Boolean.valueOf((compareValues(leftValue, rightValue) >= 0)); - default: - throw new ExpressionEvaluationException("ERR.015.006.0012", QueryPlugin.Util.getString("ERR.015.006.0012", criteria.getOperator())); //$NON-NLS-1$ //$NON-NLS-2$ - } - } - - private static final int compareValues(Object leftValue, Object rightValue) { - assert leftValue instanceof Comparable<?>; - assert rightValue instanceof Comparable<?>; - if (leftValue == rightValue) { - return 0; - } - return Constant.compare((Comparable<?>)leftValue, (Comparable<?>)rightValue); - } - public Boolean evaluate(MatchCriteria criteria, List<?> tuple) throws ExpressionEvaluationException, BlockedException, TeiidComponentException { @@ -467,7 +438,7 @@ } if(value != null) { - if(compareValues(leftValue, value) == 0) { + if(Constant.COMPARATOR.compare(leftValue, value) == 0) { return Boolean.valueOf(!criteria.isNegated()); } // else try next value } else { @@ -532,30 +503,7 @@ Object value = valueIter.next(); if(value != null) { - int compare = compareValues(leftValue, value); - // Compare two non-null values using specified operator - switch(criteria.getOperator()) { - case SubqueryCompareCriteria.EQ: - result = Boolean.valueOf(compare == 0); - break; - case SubqueryCompareCriteria.NE: - result = Boolean.valueOf(compare != 0); - break; - case SubqueryCompareCriteria.LT: - result = Boolean.valueOf(compare < 0); - break; - case SubqueryCompareCriteria.LE: - result = Boolean.valueOf(compare <= 0); - break; - case SubqueryCompareCriteria.GT: - result = Boolean.valueOf(compare > 0); - break; - case SubqueryCompareCriteria.GE: - result = Boolean.valueOf(compare >= 0); - break; - default: - throw new ExpressionEvaluationException("ERR.015.006.0012", QueryPlugin.Util.getString("ERR.015.006.0012", criteria.getOperator())); //$NON-NLS-1$ //$NON-NLS-2$ - } + result = compare(criteria, leftValue, value); switch(criteria.getPredicateQuantifier()) { case SubqueryCompareCriteria.ALL: @@ -582,6 +530,36 @@ return result; } + public static Boolean compare(AbstractCompareCriteria criteria, Object leftValue, + Object value) throws AssertionError { + int compare = Constant.COMPARATOR.compare(leftValue, value); + // Compare two non-null values using specified operator + Boolean result = null; + switch(criteria.getOperator()) { + case CompareCriteria.EQ: + result = Boolean.valueOf(compare == 0); + break; + case CompareCriteria.NE: + result = Boolean.valueOf(compare != 0); + break; + case CompareCriteria.LT: + result = Boolean.valueOf(compare < 0); + break; + case CompareCriteria.LE: + result = Boolean.valueOf(compare <= 0); + break; + case CompareCriteria.GT: + result = Boolean.valueOf(compare > 0); + break; + case CompareCriteria.GE: + result = Boolean.valueOf(compare >= 0); + break; + default: + throw new AssertionError(); + } + return result; + } + public boolean evaluate(ExistsCriteria criteria, List<?> tuple) throws BlockedException, TeiidComponentException, ExpressionEvaluationException { Modified: trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -749,13 +749,21 @@ // ================== Function = righttrim ===================== public static String rightTrim(String string, char trimChar) { + return rightTrim(string, trimChar, true); + } + + public static String rightTrim(String string, char trimChar, boolean newString) { for(int i=string.length()-1; i>=0; i--) { if(string.charAt(i) != trimChar) { // end of trim, return what's left if (i==string.length()-1) { return string; } - return new String(string.substring(0, i+1)); + String result = string.substring(0, i+1); + if (newString) { + return new String(result); + } + return result; } } Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -100,11 +100,11 @@ setState.valueIterator = dvs.getValueIterator(setState.valueExpression); if (setState.maxNdv > 0 && setState.maxNdv < dvs.getTupleBuffer().getRowCount()) { ValueIterator vi = dvs.getValueIterator(setState.valueExpression); - Comparable last = null; + Object last = null; int distinctCount = 0; while (vi.hasNext()) { - Comparable next = (Comparable) vi.next(); - if (last == null || Constant.compare(next, last) != 0) { + Object next = vi.next(); + if (last == null || Constant.COMPARATOR.compare(next, last) != 0) { distinctCount++; } last = next; Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/ListNestedSortComparator.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -174,7 +174,7 @@ return -1; } } else { - compare = Constant.compare(param1, param2); + compare = Constant.COMPARATOR.compare(param1, param2); } if (compare != 0) { boolean asc = orderTypes != null?orderTypes.get(k):this.ascendingOrder; Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/MergeJoinStrategy.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -323,7 +323,7 @@ return 1; } - int c = Constant.compare((Comparable)rightValue, (Comparable)leftValue); + int c = Constant.COMPARATOR.compare(rightValue, leftValue); if (c != 0) { return c; } Modified: trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/java/org/teiid/query/sql/symbol/Constant.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -23,11 +23,17 @@ package org.teiid.query.sql.symbol; import java.math.BigDecimal; +import java.text.Collator; +import java.util.Comparator; import java.util.List; +import java.util.Locale; import org.teiid.core.types.ClobType; import org.teiid.core.types.DataTypeManager; +import org.teiid.logging.LogConstants; +import org.teiid.logging.LogManager; import org.teiid.query.QueryPlugin; +import org.teiid.query.function.FunctionMethods; import org.teiid.query.sql.LanguageVisitor; import org.teiid.query.sql.visitor.SQLStringVisitor; @@ -47,6 +53,72 @@ private boolean multiValued; private boolean bindEligible; + public static final String COLLATION_LOCALE = System.getProperties().getProperty("org.teiid.collationLocale"); //$NON-NLS-1$ + + public static final Comparator<Object> COMPARATOR = getComparator(COLLATION_LOCALE, DataTypeManager.PAD_SPACE); + + static Comparator<Object> getComparator(String localeString, final boolean padSpace) { + if (localeString == null) { + return getComparator(padSpace); + } + String[] parts = localeString.split("_"); //$NON-NLS-1$ + Locale locale = null; + if (parts.length == 1) { + locale = new Locale(parts[0]); + } else if (parts.length == 2) { + locale = new Locale(parts[0], parts[1]); + } else if (parts.length == 3) { + locale = new Locale(parts[0], parts[1], parts[2]); + } else { + LogManager.logError(LogConstants.CTX_DQP, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30032, localeString)); + return getComparator(padSpace); + } + final Collator c = Collator.getInstance(locale); + LogManager.logError(LogConstants.CTX_DQP, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30033, locale)); + return new Comparator<Object>() { + @Override + public int compare(Object o1, Object o2) { + Class<?> clazz = o1.getClass(); + if (clazz == String.class) { + String s1 = (String)o1; + String s2 = (String)o2; + if (padSpace) { + s1 = FunctionMethods.rightTrim(s1, ' ', false); + s2 = FunctionMethods.rightTrim(s2, ' ', false); + } + return c.compare(s1, s2); + } + return ((Comparable<Object>)o1).compareTo(o2); + } + }; + } + + static Comparator<Object> getComparator(boolean padSpace) { + if (!padSpace) { + return new Comparator<Object>() { + @Override + public int compare(Object o1, Object o2) { + return ((Comparable<Object>)o1).compareTo(o2); + } + }; + } + return new Comparator<Object>() { + @Override + public int compare(Object o1, Object o2) { + Class<?> clazz = o1.getClass(); + if (clazz == String.class) { + CharSequence s1 = (CharSequence)o1; + CharSequence s2 = (CharSequence)o2; + return comparePadded(s1, s2); + } else if (clazz == ClobType.class) { + CharSequence s1 = ((ClobType)o1).getCharSequence(); + CharSequence s2 = ((ClobType)o2).getCharSequence(); + return comparePadded(s1, s2); + } + return ((Comparable<Object>)o1).compareTo(o2); + } + }; + } /** * Construct a typed constant. The specified value is not verified to be a value * of the specified type. If this is not true, stuff probably won't work later on. @@ -224,31 +296,9 @@ if (o.isNull()) { return 1; } - return compare((Comparable<?>)this.value, (Comparable<?>)o.getValue()); + return COMPARATOR.compare(this.value, o.getValue()); } - /** - * Compare the given non-null values - * @param o1 - * @param o2 - * @return - */ - public final static int compare(Comparable o1, Comparable o2) { - if (DataTypeManager.PAD_SPACE) { - Class<?> clazz = o1.getClass(); - if (clazz == String.class) { - CharSequence s1 = (CharSequence)o1; - CharSequence s2 = (CharSequence)o2; - return comparePadded(s1, s2); - } else if (clazz == ClobType.class) { - CharSequence s1 = ((ClobType)o1).getCharSequence(); - CharSequence s2 = ((ClobType)o2).getCharSequence(); - return comparePadded(s1, s2); - } - } - return o1.compareTo(o2); - } - final static int comparePadded(CharSequence s1, CharSequence s2) { int len1 = s1.length(); int len2 = s2.length(); Modified: trunk/engine/src/main/resources/org/teiid/query/i18n.properties =================================================================== --- trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2012-01-31 02:07:37 UTC (rev 3835) @@ -78,7 +78,6 @@ # processor (006) ERR.015.006.0010= Unknown criteria type: {0} ERR.015.006.0011= Unable to evaluate {0} expression of {1} -ERR.015.006.0012= Unknown compare criteria operator: {0} ERR.015.006.0014= Failed to create regular expression from match pattern: {0}. {1} ERR.015.006.0015= Unable to evaluate expression of {0} ERR.015.006.0016= Unknown expression type: {0} @@ -956,3 +955,6 @@ TEIID30026=Failed to cancel {0} TEIID30030=Unhandled exception disposing reusable execution TEIID30031=Unhandled exception calling CommandListener +TEIID30032=Invalid locale {0} for collation, using default collation +TEIID30033=Using collator for locale {0} + Modified: trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java 2012-01-30 18:19:21 UTC (rev 3834) +++ trunk/engine/src/test/java/org/teiid/query/sql/symbol/TestConstant.java 2012-01-31 02:07:37 UTC (rev 3835) @@ -24,6 +24,11 @@ import static org.junit.Assert.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + import org.junit.Test; import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.UnitTestUtil; @@ -155,7 +160,16 @@ assertEquals(-24, Constant.comparePadded("ab ", "az ")); assertEquals(66, Constant.comparePadded("ab ", "a ")); assertEquals(0, Constant.comparePadded("a1 ", "a1")); - } + + @Test public void testCollation() { + Comparator<Object> c = Constant.getComparator("es", true); + + List<String> vals = Arrays.asList("ñ", "n", "o"); + Collections.sort(vals, c); + assertEquals("ñ", vals.get(1)); + + assertEquals(0, c.compare("a ", "a")); + } }