Author: shawkins Date: 2010-10-29 13:21:21 -0400 (Fri, 29 Oct 2010) New Revision: 2689 Modified: trunk/api/src/main/java/org/teiid/language/SQLConstants.java trunk/connectors/translator-file/src/main/java/org/teiid/translator/file/FileExecutionFactory.java trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java trunk/engine/src/main/java/org/teiid/query/function/aggregate/TextAgg.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java trunk/engine/src/main/java/org/teiid/query/sql/symbol/DerivedColumn.java trunk/engine/src/main/java/org/teiid/query/sql/symbol/TextLine.java trunk/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj trunk/engine/src/main/resources/org/teiid/query/i18n.properties trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java trunk/metadata/src/test/java/org/teiid/metadata/index/TestMultipleModelIndexes.java Log: TEIID-962 refinements to textagg. added the for keyword so that textagg can be non-reserved. also changed to directly output a blob value and expanded the validation. Modified: trunk/api/src/main/java/org/teiid/language/SQLConstants.java =================================================================== --- trunk/api/src/main/java/org/teiid/language/SQLConstants.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/api/src/main/java/org/teiid/language/SQLConstants.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -98,6 +98,9 @@ public static final String LAST = "LAST"; //$NON-NLS-1$ public static final String KEY = "KEY"; //$NON-NLS-1$ + + public static final String ENCODING = "ENCODING"; //$NON-NLS-1$ + public static final String TEXTAGG = "TEXTAGG"; //$NON-NLS-1$ } public interface Reserved { @@ -319,7 +322,6 @@ public static final String SYSTEM_USER = "SYSTEM_USER"; //$NON-NLS-1$ public static final String TABLE = "TABLE"; //$NON-NLS-1$ public static final String TEMPORARY = "TEMPORARY"; //$NON-NLS-1$ - public static final String TEXTAGG = "TEXTAGG"; //$NON-NLS-1$ public static final String THEN = "THEN"; //$NON-NLS-1$ public static final String TIME = "TIME"; //$NON-NLS-1$ public static final String TIMESTAMP = "TIMESTAMP"; //$NON-NLS-1$ Modified: trunk/connectors/translator-file/src/main/java/org/teiid/translator/file/FileExecutionFactory.java =================================================================== --- trunk/connectors/translator-file/src/main/java/org/teiid/translator/file/FileExecutionFactory.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/connectors/translator-file/src/main/java/org/teiid/translator/file/FileExecutionFactory.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -109,7 +109,7 @@ Object value = null; if (isText) { ClobImpl clob = new ClobImpl(isf, -1); - clob.setEncoding(encoding); + clob.setEncoding(encoding.name()); value = new ClobType(clob); } else { value = new BlobType(new BlobImpl(isf)); @@ -131,15 +131,15 @@ public static final String GETFILES = "getFiles"; //$NON-NLS-1$ public static final String SAVEFILE = "saveFile"; //$NON-NLS-1$ - private String encoding = Charset.defaultCharset().name(); + private Charset encoding = Charset.defaultCharset(); @TranslatorProperty(display="File Encoding",advanced=true) public String getEncoding() { - return encoding; + return encoding.name(); } public void setEncoding(String encoding) { - this.encoding = encoding; + this.encoding = Charset.forName(encoding); } //@Override @@ -161,7 +161,7 @@ if (file instanceof SQLXML) { is = ((SQLXML)file).getBinaryStream(); } else if (file instanceof Clob) { - is = new ReaderInputStream(((Clob)file).getCharacterStream(), Charset.forName(encoding)); + is = new ReaderInputStream(((Clob)file).getCharacterStream(), encoding); } else if (file instanceof Blob) { is = ((Blob)file).getBinaryStream(); } else { Modified: trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -30,6 +30,7 @@ import java.sql.SQLException; import java.sql.SQLXML; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -649,7 +650,7 @@ } else if (expression instanceof Criteria) { return evaluate((Criteria)expression, tuple); } else if (expression instanceof TextLine){ - return evaluateTextForest(tuple, (TextLine)expression); + return evaluateTextLine(tuple, (TextLine)expression); } else if (expression instanceof XMLElement){ return evaluateXMLElement(tuple, (XMLElement)expression); } else if (expression instanceof XMLForest){ @@ -813,11 +814,19 @@ return new ClobType(new ClobImpl(isf, -1)); } - private Object evaluateTextForest(List<?> tuple, TextLine function) throws ExpressionEvaluationException, BlockedException, TeiidComponentException, FunctionExecutionException { + private Object evaluateTextLine(List<?> tuple, TextLine function) throws ExpressionEvaluationException, BlockedException, TeiidComponentException, FunctionExecutionException { List<DerivedColumn> args = function.getExpressions(); Evaluator.NameValuePair<Object>[] nameValuePairs = getNameValuePairs(tuple, args, true); - - return TextLine.evaluate(nameValuePairs, function.getDelimiter(), function.getQuote()); + + try { + return TextLine.evaluate(Arrays.asList(nameValuePairs), new TextLine.ValueExtractor<NameValuePair<Object>>() { + public Object getValue(NameValuePair<Object> t) { + return t.value; + } + }, function.getDelimiter(), function.getQuote()); + } catch (TransformationException e) { + throw new ExpressionEvaluationException(e, e.getMessage()); + } } private Object evaluateXMLForest(List<?> tuple, XMLForest function) Modified: trunk/engine/src/main/java/org/teiid/query/function/aggregate/TextAgg.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/aggregate/TextAgg.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/function/aggregate/TextAgg.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -25,18 +25,21 @@ import java.io.IOException; import java.io.Writer; import java.sql.SQLException; +import java.util.Arrays; import java.util.List; -import javax.sql.rowset.serial.SerialClob; +import javax.sql.rowset.serial.SerialBlob; import org.teiid.common.buffer.FileStore; import org.teiid.common.buffer.FileStore.FileStoreOutputStream; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; -import org.teiid.core.types.ClobImpl; -import org.teiid.core.types.ClobType; +import org.teiid.core.types.BlobImpl; +import org.teiid.core.types.BlobType; import org.teiid.core.types.Streamable; import org.teiid.query.processor.xml.XMLUtil.FileStoreInputStreamFactory; +import org.teiid.query.sql.symbol.DerivedColumn; +import org.teiid.query.sql.symbol.ElementSymbol; import org.teiid.query.sql.symbol.TextLine; import org.teiid.query.util.CommandContext; @@ -54,13 +57,20 @@ this.textLine = textLine; } - private FileStoreInputStreamFactory buildResult(CommandContext context, TextLine textLine) throws TeiidProcessingException { + private FileStoreInputStreamFactory buildResult() throws TeiidProcessingException { try { FileStore fs = context.getBufferManager().createFileStore("textagg"); //$NON-NLS-1$ - FileStoreInputStreamFactory fisf = new FileStoreInputStreamFactory(fs, Streamable.ENCODING); + FileStoreInputStreamFactory fisf = new FileStoreInputStreamFactory(fs, textLine.getEncoding()==null?Streamable.ENCODING:textLine.getEncoding()); Writer w = fisf.getWriter(); if (textLine.isIncludeHeader()) { - w.write(TextLine.getHeader(textLine.getExpressions(), textLine.getDelimiter(), textLine.getQuote())); + w.write(TextLine.evaluate(textLine.getExpressions(), new TextLine.ValueExtractor<DerivedColumn>() { + public Object getValue(DerivedColumn t) { + if (t.getAlias() == null && t.getExpression() instanceof ElementSymbol) { + return ((ElementSymbol)t.getExpression()).getShortName(); + } + return t.getAlias(); + } + }, textLine.getDelimiter(), textLine.getQuote())); } w.close(); return fisf; @@ -81,7 +91,7 @@ public void addInputDirect(Object input, List<?> tuple) throws TeiidComponentException, TeiidProcessingException { try { if (this.result == null) { - this.result = buildResult(this.context, this.textLine); + this.result = buildResult(); } String in = (String)input; Writer w = result.getWriter(); @@ -97,7 +107,7 @@ */ public Object getResult() throws TeiidProcessingException{ if (this.result == null) { - this.result = buildResult(this.context, this.textLine); + this.result = buildResult(); } try { @@ -105,11 +115,9 @@ fs.close(); if (fs.bytesWritten()) { - return new ClobType(new ClobImpl(result, -1)); + return new BlobType(new BlobImpl(result)); } - // fun convert bytes to string to char array!! - String msg = new String(fs.getBuffer(),0, fs.getCount(), Streamable.ENCODING); - return new ClobType(new SerialClob(msg.toCharArray())); + return new BlobType(new SerialBlob(Arrays.copyOf(fs.getBuffer(), fs.getCount()))); } catch (IOException e) { throw new TeiidProcessingException(e); } catch (SQLException e) { Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -119,7 +119,7 @@ @Override public void visit(TextLine obj) { - markInvalid(obj, "Pushdown of TextForest not allowed"); //$NON-NLS-1$ + markInvalid(obj, "Pushdown of TextLine not allowed"); //$NON-NLS-1$ } @Override Modified: trunk/engine/src/main/java/org/teiid/query/sql/symbol/DerivedColumn.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/symbol/DerivedColumn.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/sql/symbol/DerivedColumn.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -24,6 +24,7 @@ import org.teiid.core.util.EquivalenceUtil; import org.teiid.core.util.HashCodeUtil; +import org.teiid.query.function.source.XMLSystemFunctions; import org.teiid.query.sql.LanguageObject; import org.teiid.query.sql.LanguageVisitor; import org.teiid.query.sql.visitor.SQLStringVisitor; Modified: trunk/engine/src/main/java/org/teiid/query/sql/symbol/TextLine.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/symbol/TextLine.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/sql/symbol/TextLine.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -21,23 +21,30 @@ */ package org.teiid.query.sql.symbol; -import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.teiid.core.types.DataTypeManager; +import org.teiid.core.types.TransformationException; import org.teiid.core.util.EquivalenceUtil; import org.teiid.core.util.HashCodeUtil; -import org.teiid.query.eval.Evaluator; +import org.teiid.core.util.StringUtil; +import org.teiid.query.sql.LanguageObject; import org.teiid.query.sql.LanguageVisitor; import org.teiid.query.sql.visitor.SQLStringVisitor; +/** + * Represents the only allowable expression for the textagg aggregate. + * This is a Teiid specific construct. + */ public class TextLine implements Expression { - public static String nl = System.getProperty("line.separator"); //$NON-NLS-1$ + public static final String nl = System.getProperty("line.separator"); //$NON-NLS-1$ - private Character delimiter = null; - private Character quote = null; + private Character delimiter; + private Character quote; private boolean includeHeader; private List<DerivedColumn> expressions; + private String encoding; public Character getDelimiter() { return delimiter; @@ -46,6 +53,14 @@ public void setDelimiter(Character delimiter) { this.delimiter = delimiter; } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } public Character getQuote() { return quote; @@ -73,7 +88,7 @@ @Override public Class<?> getType() { - return DataTypeManager.DefaultDataClasses.CLOB; + return DataTypeManager.DefaultDataClasses.BLOB; } @Override @@ -94,24 +109,11 @@ @Override public TextLine clone() { TextLine clone = new TextLine(); - - if (this.expressions != null && !this.expressions.isEmpty()) { - List<DerivedColumn> list = new ArrayList<DerivedColumn>(); - for (DerivedColumn expr:this.expressions) { - list.add(expr.clone()); - } - clone.expressions = list; - } - - if (this.delimiter != null) { - clone.delimiter = new Character(this.delimiter); - } - - if (this.quote != null) { - clone.quote = new Character(this.quote); - } - + clone.expressions = LanguageObject.Util.deepClone(this.expressions, DerivedColumn.class); + clone.delimiter = this.delimiter; + clone.quote = this.quote; clone.includeHeader = this.includeHeader; + clone.encoding = this.encoding; return clone; } @@ -127,7 +129,8 @@ return EquivalenceUtil.areEqual(this.expressions, other.expressions) && EquivalenceUtil.areEqual(this.delimiter, other.delimiter) && EquivalenceUtil.areEqual(this.quote, other.quote) - && this.includeHeader == other.includeHeader; + && this.includeHeader == other.includeHeader + && EquivalenceUtil.areEqual(this.encoding, other.encoding); } @Override @@ -140,57 +143,37 @@ return SQLStringVisitor.getSQLString(this); } - public static String evaluate(final Evaluator.NameValuePair[] values, Character delimeter, Character quote) { - + public static interface ValueExtractor<T> { + Object getValue(T t); + } + + public static <T> String evaluate(final List<T> values, ValueExtractor<T> valueExtractor, Character delimeter, Character quote) throws TransformationException { if (delimeter == null) { delimeter = new Character(','); } - + String quoteStr = null; + if (quote == null) { + quoteStr = "\""; //$NON-NLS-1$ + } else { + quoteStr = String.valueOf(quote); + } + String doubleQuote = quoteStr + quoteStr; StringBuilder sb = new StringBuilder(); - for (int i = 0; i < values.length; i++) { - if (values[i].value != null) { - addQuote(quote, sb); - sb.append(values[i].value); - addQuote(quote, sb); + for (Iterator<T> iterator = values.iterator(); iterator.hasNext();) { + T t = iterator.next(); + String text = DataTypeManager.transformValue(valueExtractor.getValue(t), DataTypeManager.DefaultDataClasses.STRING); + if (text == null) { + continue; } - if (i < values.length-1) { + sb.append(quoteStr); + sb.append(StringUtil.replaceAll(text, quoteStr, doubleQuote)); + sb.append(quoteStr); + if (iterator.hasNext()) { sb.append(delimeter); } } sb.append(nl); - return sb.toString(); } - - public static String getHeader(List<DerivedColumn> args, Character delimeter, Character quote) { - - if (delimeter == null) { - delimeter = new Character(','); - } - - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < args.size(); i++) { - DerivedColumn symbol = args.get(i); - String name = symbol.getAlias(); - Expression ex = symbol.getExpression(); - if (name == null && ex instanceof ElementSymbol) { - name = ((ElementSymbol)ex).getShortName(); - } - addQuote(quote, sb); - sb.append(name); - addQuote(quote, sb); - - if (i < args.size()-1) { - sb.append(delimeter); - } - } - sb.append(nl); - return sb.toString(); - } - private static void addQuote(Character quote, StringBuilder sb) { - if (quote != null) { - sb.append(quote); - } - } } Modified: trunk/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -1595,8 +1595,8 @@ @Override public void visit( TextLine obj ) { - append("TEXTLINE"); //$NON-NLS-1$ - append("("); //$NON-NLS-1$ + append(FOR); + append(SPACE); registerNodes(obj.getExpressions(), 0); if (obj.getDelimiter() != null) { @@ -1615,8 +1615,12 @@ append(SPACE); append(NonReserved.HEADER); } - - append(")"); //$NON-NLS-1$ + if (obj.getEncoding() != null) { + append(SPACE); + append(NonReserved.ENCODING); + append(SPACE); + outputDisplayName(obj.getEncoding()); + } } @Override Modified: trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -22,6 +22,7 @@ package org.teiid.query.validator; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -124,6 +125,7 @@ import org.teiid.query.sql.symbol.XMLNamespaces; import org.teiid.query.sql.symbol.XMLParse; import org.teiid.query.sql.symbol.XMLQuery; +import org.teiid.query.sql.symbol.AggregateSymbol.Type; import org.teiid.query.sql.util.SymbolMap; import org.teiid.query.sql.visitor.AggregateSymbolCollectorVisitor; import org.teiid.query.sql.visitor.CommandCollectorVisitor; @@ -1241,17 +1243,27 @@ } @Override - public void visit(TextLine obj) { - validateDerivedColumnNames(obj, obj.getExpressions()); - for (DerivedColumn dc : obj.getExpressions()) { - if (dc.getAlias() == null) { - continue; - } - validateQName(obj, dc.getAlias()); + public void visit(AggregateSymbol obj) { + if (obj.getAggregateFunction() != Type.TEXTAGG) { + return; + } + TextLine tl = (TextLine)obj.getExpression(); + if (tl.isIncludeHeader()) { + validateDerivedColumnNames(obj, tl.getExpressions()); + } + for (DerivedColumn dc : tl.getExpressions()) { validateXMLContentTypes(dc.getExpression(), obj); } + validateTextOptions(obj, tl.getDelimiter(), tl.getQuote()); + if (tl.getEncoding() != null) { + try { + Charset.forName(tl.getEncoding()); + } catch (IllegalArgumentException e) { + handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.invalid_encoding", tl.getEncoding()), obj); //$NON-NLS-1$ + } + } } - + private String[] validateQName(LanguageObject obj, String name) { try { return Name11Checker.getInstance().getQNameParts(name); @@ -1401,31 +1413,34 @@ handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_width"), obj); //$NON-NLS-1$ } } else { - delimiter = obj.getDelimiter(); - if (delimiter == null) { - delimiter = ','; - } + if (obj.getHeader() != null && obj.getHeader() < 0) { + handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_negative"), obj); //$NON-NLS-1$ + } + delimiter = obj.getDelimiter(); + quote = obj.getQuote(); + validateTextOptions(obj, delimiter, quote); } if (obj.getSkip() != null && obj.getSkip() < 0) { handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_negative"), obj); //$NON-NLS-1$ } - if (!widthSet) { - if (obj.getHeader() != null && obj.getHeader() < 0) { - handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_negative"), obj); //$NON-NLS-1$ - } - quote = obj.getQuote(); - if (quote == null) { - quote = '"'; - } - if (EquivalenceUtil.areEqual(quote, delimiter)) { - handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_delimiter"), obj); //$NON-NLS-1$ - } - if (EquivalenceUtil.areEqual(quote, '\n') - || EquivalenceUtil.areEqual(delimiter, '\n')) { - handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_newline"), obj); //$NON-NLS-1$ - } + } + + private void validateTextOptions(LanguageObject obj, Character delimiter, + Character quote) { + if (quote == null) { + quote = '"'; + } + if (delimiter == null) { + delimiter = ','; } - } + if (EquivalenceUtil.areEqual(quote, delimiter)) { + handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_delimiter"), obj); //$NON-NLS-1$ + } + if (EquivalenceUtil.areEqual(quote, '\n') + || EquivalenceUtil.areEqual(delimiter, '\n')) { + handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.text_table_newline"), obj); //$NON-NLS-1$ + } + } @Override public void visit(XMLParse obj) { Modified: trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj =================================================================== --- trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj 2010-10-29 17:21:21 UTC (rev 2689) @@ -260,7 +260,6 @@ | <SYSTEM_USER: "system_user"> | <TABLE: "table"> | <TEMPORARY: "temporary"> -| <TEXTAGG: "textagg"> | <THEN: "then"> | <TIMEZONE_HOUR: "timezone_hour"> | <TIMEZONE_MINUTE: "timezone_minute"> @@ -1795,13 +1794,14 @@ DerivedColumn expression = null; Character delimiter = null; Character quote = null; - Integer header = null; + boolean header = false; List<DerivedColumn> expressions = new ArrayList<DerivedColumn>(); OrderBy orderBy = null; + String encoding = null; } { - <TEXTAGG> <LPAREN> - expression = derivedColumn(info) + nonReserved("TEXTAGG") <LPAREN> + <FOR> expression = derivedColumn(info) { expressions.add(expression); } @@ -1821,16 +1821,13 @@ ] [ LOOKAHEAD(<ID>, { "header".equalsIgnoreCase(getToken(1).image) }) <ID> - [ - header = intVal() - ] - { - if (header == null) { - header = 1; - } - } + { header = true; } ] [ + LOOKAHEAD(<ID>, { "encoding".equalsIgnoreCase(getToken(1).image) }) + ( <ID> encoding = id() ) + ] + [ orderBy = orderby(info) ] <RPAREN> @@ -1842,9 +1839,9 @@ TextLine tf = new TextLine(); tf.setDelimiter(delimiter); tf.setQuote(quote); - tf.setIncludeHeader(header!=null); + tf.setIncludeHeader(header); tf.setExpressions(expressions); - + tf.setEncoding(encoding); String name = generateFunctionName(info, "TEXTAGG"); AggregateSymbol agg = new AggregateSymbol(name, "TEXTAGG", false, tf); agg.setOrderBy(orderBy); @@ -3199,6 +3196,8 @@ <RBRACE> ) | + LOOKAHEAD(<ID> <LPAREN> <FOR>) (expression=textAgg(info)) + | // Aggregate function LOOKAHEAD(<ID>, {matchesAny(getToken(1).image, "count", "min", "max", "sum", "avg", "every", "STDDEV_POP", "STDDEV_SAMP", "VAR_SAMP", "VAR_POP") != null}) (expression=aggregateSymbol(info)) | @@ -3208,8 +3207,6 @@ | (expression=xmlAgg(info)) | - (expression=textAgg(info)) - | // Function LOOKAHEAD(2) (expression=function(info)) | Modified: trunk/engine/src/main/resources/org/teiid/query/i18n.properties =================================================================== --- trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2010-10-29 17:21:21 UTC (rev 2689) @@ -187,13 +187,14 @@ ERR.015.012.0025 = Command must project at least one symbol ERR.015.012.0026 = Expressions of type OBJECT, CLOB, BLOB, or XML cannot be used in SELECT DISTINCT, ORDER BY, GROUP BY, KEYS, or non-all set queries: [{0}] ERR.015.012.0027 = Expressions of type OBJECT, CLOB, BLOB, or XML cannot be used in comparison: {0}. -ValidationVisitor.expression_requires_name = Non-column expressions require a name in XMLATTRIBUTES, XMLFOREST, or QUERYSTRING +ValidationVisitor.expression_requires_name = Non-column expressions require a name in XMLATTRIBUTES, XMLFOREST, QUERYSTRING, or TEXTAGG with HEADER ValidationVisitor.invalid_lookup_key=Expressions of type OBJECT, CLOB, BLOB, or XML cannot be used as LOOKUP key columns: {0}. ValidationVisitor.limit_not_valid_for_xml=The limit clause cannot be used on an XML document query. ValidationVisitor.translated_or=Translated user criteria must not contain OR criteria ValidationVisitor.union_insert = Select into is not allowed under a set operation: {0}. ValidationVisitor.multisource_insert = A multi-source table, {0}, cannot be used in an INSERT with query expression or SELECT INTO statement. ValidationVisitor.virtual_update_subquery = Subqueries are not allowed in the criteria for a virtual UPDATE/DELETE: {0} +ValidationVisitor.invalid_encoding = Invalid encoding: {0}. ERR.015.012.0029 = INSERT, UPDATE, and DELETE not allowed on XML documents ERR.015.012.0030 = Commands used in stored procedure language not allowed on XML documents ERR.015.012.0031 = Queries against XML documents can not have a GROUP By clause @@ -688,13 +689,13 @@ ValidationVisitor.text_table_invalid_width=For a fixed width text table, all columns must have width set. ValidationVisitor.text_table_width=Fixed width text tables should not have DELIMITER, QUOTE, ESCAPE, or HEADER specified. ValidationVisitor.text_table_negative=Text table WIDTH, HEADER, or SKIP values must not be negative. -ValidationVisitor.text_table_delimiter=Text table DELIMITER cannot be the same as the QUOTE or ESCAPE characters. -ValidationVisitor.text_table_newline=Text table DELIMITER, QUOTE, and ESCAPE characters cannot be the new line character. +ValidationVisitor.text_table_delimiter=Text DELIMITER cannot be the same as the QUOTE or ESCAPE characters. +ValidationVisitor.text_table_newline=Text DELIMITER, QUOTE, and ESCAPE characters cannot be the new line character. ValidationVisitor.xml_namespaces=At most only one NO DEFAULT or DEFAULT namespace may be specified. ValidationVisitor.xml_namespaces_reserved=The namespaces xmlns and xml are reserved. ValidationVisitor.xml_namespaces_null_uri=The null uri, or empty string, is not allowed as the uri value. ValidationVisitor.xml_attributes_reserved=The namespace xmlns is reserved. -ValidationVisitor.xml_content_type=The expression "{0}" is of OBJECT or BLOB type, which cannot be used as an XML content value. +ValidationVisitor.xml_content_type=The expression "{0}" is of OBJECT or BLOB type, which cannot be used as an XML or TEXT content value. ValidationVisitor.xml_invalid_qname=The qname "{0}" is invalid. ValidationVisitor.context_item_type=XMLTABLE or XMLQUERY PASSING context item must be an XML value. ValidationVisitor.passing_requires_name=XMLTABLE or XMLQUERY PASSING clause can only contain at most 1 unnamed item. Modified: trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -40,6 +40,7 @@ import org.teiid.client.metadata.ParameterInfo; import org.teiid.core.TeiidException; import org.teiid.core.types.DataTypeManager; +import org.teiid.language.SQLConstants.NonReserved; import org.teiid.language.SQLConstants.Reserved; import org.teiid.language.SortSpecification.NullOrdering; import org.teiid.query.sql.lang.BetweenCriteria; @@ -6725,29 +6726,16 @@ tf.setDelimiter(new Character(',')); tf.setIncludeHeader(true); - AggregateSymbol as = new AggregateSymbol("foo", Reserved.TEXTAGG, false, tf); + AggregateSymbol as = new AggregateSymbol("foo", NonReserved.TEXTAGG, false, tf); as.setOrderBy(new OrderBy(Arrays.asList(new ElementSymbol("e2")))); Query query = new Query(); query.setSelect(new Select(Arrays.asList(as))); - String sql = "SELECT TextAgg(e1 as col1, e2 as col2 delimiter ',' header order by e2)"; //$NON-NLS-1$ - helpTest(sql, "SELECT TEXTAGG(TEXTLINE(e1 AS col1, e2 AS col2 DELIMITER ',' HEADER) ORDER BY e2)", query); + String sql = "SELECT TextAgg(FOR e1 as col1, e2 as col2 delimiter ',' header order by e2)"; //$NON-NLS-1$ + helpTest(sql, "SELECT TEXTAGG(FOR e1 AS col1, e2 AS col2 DELIMITER ',' HEADER ORDER BY e2)", query); } -// @Test public void testTextForrest() throws Exception { -// List<DerivedColumn> expressions = new ArrayList<DerivedColumn>(); -// expressions.add(new DerivedColumn(null, new Constant(1))); -// expressions.add(new DerivedColumn("col2", new ElementSymbol("e2"))); -// -// TextForest tf = new TextForest(); -// tf.setExpressions(expressions); -// tf.setIncludeHeader(true); -// -// String sql = "textforest(1, e2 as col2 HEADER)"; //$NON-NLS-1$ -// helpTestExpression(sql, "TEXTFOREST(1, e2 AS col2 HEADER)", tf); -// } - @Test public void testNestedTable() throws Exception { String sql = "SELECT * from TABLE(exec foo()) as x"; //$NON-NLS-1$ Query query = new Query(); Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -239,7 +239,7 @@ caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_MAX, true); capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ - ProcessorPlan plan = helpPlan("select convert(textagg(pm1.g1.e1, pm1.g1.e2 header order by e2), string) as x from pm1.g1", metadata, null, capFinder, //$NON-NLS-1$ + ProcessorPlan plan = helpPlan("select convert(to_chars(textagg(for pm1.g1.e1, pm1.g1.e2 header order by e2), 'UTF-8'), string) as x from pm1.g1", metadata, null, capFinder, //$NON-NLS-1$ new String[] { "SELECT g_0.e1, g_0.e2 FROM pm1.g1 AS g_0" }, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$ HardcodedDataManager hdm = new HardcodedDataManager(); @@ -248,7 +248,7 @@ String nl = System.getProperty("line.separator"); ArrayList list = new ArrayList(); - list.add("e1,e2"+nl+"b,1"+nl+"z,2"+nl); + list.add("\"e1\",\"e2\""+nl+"\"b\",\"1\""+nl+"\"z\",\"2\""+nl); List[] expected = new List[] { list, }; Modified: trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -2083,5 +2083,13 @@ models.add("pm1"); helpValidate("insert into pm1.g1 select * from pm1.g1", new String[] {"pm1.g1"}, new MultiSourceMetadataWrapper(FakeMetadataFactory.example1(), models)); //$NON-NLS-1$ } + + @Test public void testTextAggEncoding() throws Exception { + helpValidate("select textagg(for e1 encoding abc) from pm1.g1", new String[] {"TEXTAGG(FOR e1 ENCODING abc)"}, FakeMetadataFactory.example1Cached()); //$NON-NLS-1$ + } + + @Test public void testTextAggHeader() throws Exception { + helpValidate("select textagg(for e1 || 1 HEADER) from pm1.g1", new String[] {"TEXTAGG(FOR (e1 || 1) HEADER)"}, FakeMetadataFactory.example1Cached()); //$NON-NLS-1$ + } } Modified: trunk/metadata/src/test/java/org/teiid/metadata/index/TestMultipleModelIndexes.java =================================================================== --- trunk/metadata/src/test/java/org/teiid/metadata/index/TestMultipleModelIndexes.java 2010-10-28 21:21:51 UTC (rev 2688) +++ trunk/metadata/src/test/java/org/teiid/metadata/index/TestMultipleModelIndexes.java 2010-10-29 17:21:21 UTC (rev 2689) @@ -42,7 +42,7 @@ assertEquals(1, names.size()); //ensure that datatypes are set - Table t = (Table)tm.getGroupID(names.iterator().next()); + Table t = tm.getGroupID(names.iterator().next()); assertNotNull(t.getColumns().get(0).getDatatype()); }