Author: shawkins Date: 2010-06-17 21:43:01 -0400 (Thu, 17 Jun 2010) New Revision: 2243 Added: trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java Removed: trunk/documentation/quick-start-example/src/main/docbook/en-US/images/EmbeddedDesignerExampleVideo.swf Modified: trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj trunk/engine/src/main/resources/org/teiid/query/i18n.properties trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java Log: TEIID-1021 refining support for clob<->blob conversions. changes encode/decode to to_bytes/to_chars to cover both binary and character encodings. Modified: trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java =================================================================== --- trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -40,8 +40,6 @@ public static final String ASCII = "ascii"; //$NON-NLS-1$ public static final String CHAR = "char"; //$NON-NLS-1$ public static final String CONCAT = "concat"; //$NON-NLS-1$ - public static final String DECODE = "decode"; //$NON-NLS-1$ - public static final String ENCODE = "encode"; //$NON-NLS-1$ //public static final String DIFFERENCE = "difference"; public static final String INITCAP = "initcap"; //$NON-NLS-1$ public static final String INSERT = "insert"; //$NON-NLS-1$ @@ -58,6 +56,8 @@ public static final String RTRIM = "rtrim"; //$NON-NLS-1$ //public static final String SOUNDEX = "soundex"; public static final String SUBSTRING = "substring"; //$NON-NLS-1$ + public static final String TO_BYTES = "to_bytes"; //$NON-NLS-1$ + public static final String TO_CHARS = "to_chars"; //$NON-NLS-1$ public static final String UCASE = "ucase"; //$NON-NLS-1$ //numeric Modified: trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -1,3 +1,25 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + package org.teiid.core.types; import java.io.Externalizable; @@ -3,5 +25,4 @@ import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -11,13 +32,13 @@ import java.sql.SQLException; import org.teiid.core.types.InputStreamFactory.StreamFactoryReference; +import org.teiid.core.util.InputStreamReader; - public class BaseLob implements Externalizable, StreamFactoryReference { private static final long serialVersionUID = -1586959324208959519L; private InputStreamFactory streamFactory; - private String encoding; + private Charset charset; public BaseLob() { @@ -38,14 +59,22 @@ return streamFactory; } - public String getEncoding() { - return encoding; + public void setEncoding(String encoding) { + if (encoding != null) { + this.charset = Charset.forName(encoding); + } else { + this.charset = null; + } } - public void setEncoding(String encoding) { - this.encoding = encoding; + public Charset getCharset() { + return charset; } + public void setCharset(Charset charset) { + this.charset = charset; + } + public void free() throws SQLException { if (this.streamFactory != null) { try { @@ -70,11 +99,11 @@ ex.initCause(e); throw ex; } - String enc = getEncoding(); - if (enc == null) { - enc = Streamable.ENCODING; + Charset cs = getCharset(); + if (cs == null) { + cs = Streamable.CHARSET; } - return new InputStreamReader(getBinaryStream(), Charset.forName(enc)); + return new InputStreamReader(getBinaryStream(), cs.newDecoder()); } public InputStream getBinaryStream() throws SQLException { Modified: trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -84,19 +84,19 @@ } @Override - public String getEncoding() { - String enc = super.getEncoding(); - if (enc != null) { - return enc; + public Charset getCharset() { + Charset cs = super.getCharset(); + if (cs != null) { + return cs; } try { - enc = XMLType.getEncoding(this.getBinaryStream()); + String enc = XMLType.getEncoding(this.getBinaryStream()); if (enc != null) { setEncoding(enc); } } catch (SQLException e) { } - return Streamable.ENCODING; + return Streamable.CHARSET; } @SuppressWarnings("unchecked") Modified: trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.nio.charset.Charset; import org.teiid.core.CorePlugin; @@ -43,6 +44,7 @@ private static final long serialVersionUID = -8252488562134729374L; public static final String ENCODING = "UTF-8"; //$NON-NLS-1$ + public static final Charset CHARSET = Charset.forName(ENCODING); public static final String FORCE_STREAMING = "FORCE_STREAMING"; //$NON-NLS-1$ public static final int STREAMING_BATCH_SIZE_IN_BYTES = 102400; // 100K Modified: trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -29,6 +29,7 @@ import java.io.OutputStream; import java.io.Reader; import java.io.Writer; +import java.nio.charset.Charset; import java.sql.SQLException; import java.sql.SQLXML; @@ -147,7 +148,10 @@ xml = type.reference; } if (xml instanceof SQLXMLImpl) { - return ((SQLXMLImpl)xml).getEncoding(); + Charset cs = ((SQLXMLImpl)xml).getCharset(); + if (cs != null) { + return cs.displayName(); + } } return getEncoding(xml.getBinaryStream()); } catch (SQLException e) { Added: trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java (rev 0) +++ trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -0,0 +1,104 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.core.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; + +import org.teiid.core.types.DataTypeManager; + +/** + * Replacement for the standard {@link java.io.InputStreamReader}, + * which suffers from a <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4744247&quo... in sun.nio.cs.StreamDecoder + */ +public class InputStreamReader extends Reader { + + private CharsetDecoder cd; + private ReadableByteChannel rbc; + private ByteBuffer bb; + private CharBuffer cb; + private boolean done; + + public InputStreamReader(InputStream in, CharsetDecoder cd) { + this(in, cd, DataTypeManager.MAX_LOB_MEMORY_BYTES); + } + + public InputStreamReader(InputStream in, CharsetDecoder cd, int bufferSize) { + this.cd = cd; + this.rbc = Channels.newChannel(in); + this.bb = ByteBuffer.allocate(bufferSize); + this.cb = CharBuffer.allocate((int)(bufferSize * (double)cd.maxCharsPerByte())); + this.cb.limit(0); + } + + @Override + public void close() throws IOException { + rbc.close(); + cd.reset(); + } + + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + if ((off < 0) || (off > cbuf.length) || (len < 0) || + ((off + len) > cbuf.length) || ((off + len) < 0)) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + while (!done && !cb.hasRemaining()) { + int read = 0; + while ((read = rbc.read(bb)) == 0) { + //blocking read + } + bb.flip(); + cb.clear(); + CoderResult cr = cd.decode(bb, cb, read == -1); + if (!cr.isUnderflow()) { + cr.throwException(); + } + if (read == -1) { + cr = cd.flush(cb); + if (!cr.isUnderflow()) { + cr.throwException(); + } + done = true; + } + bb.clear(); + cb.flip(); + } + len = Math.min(len, cb.remaining()); + if (len == 0 && done) { + return -1; + } + cb.get(cbuf, off, len); + return len; + } + +} \ No newline at end of file Property changes on: trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -47,15 +47,15 @@ private char[] charBuffer; private AccessibleByteArrayOutputStream out = new AccessibleByteArrayOutputStream(); private boolean hasMore = true; - private int pos = 0; + private int pos; - public ReaderInputStream(Reader reader, Charset charSet) { - this(reader, charSet, DEFAULT_BUFFER_SIZE); + public ReaderInputStream(Reader reader, Charset charset) { + this(reader, charset, DEFAULT_BUFFER_SIZE); } - public ReaderInputStream(Reader reader, Charset charSet, int bufferSize) { + public ReaderInputStream(Reader reader, Charset charset, int bufferSize) { this.reader = reader; - this.writer = new OutputStreamWriter(out, charSet); + this.writer = new OutputStreamWriter(out, charset); this.charBuffer = new char[bufferSize]; } Deleted: trunk/documentation/quick-start-example/src/main/docbook/en-US/images/EmbeddedDesignerExampleVideo.swf =================================================================== (Binary files differ) Modified: trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml =================================================================== --- trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml 2010-06-18 01:43:01 UTC (rev 2243) @@ -801,27 +801,6 @@ </row> <row> <entry> - <para>ENCODE(x, encoding)</para> - </entry> - <entry> - <para>Return a clob from the blob with the given encoding. The builtin Java Charset<footnote id="charset"><para>See the <ulink url="http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charse... JavaDoc</ulink></para></footnote> names are valid values for encoding.</para> - </entry> - <entry> - <para>x is a blob, encoding is a string, and returns a clob</para> - </entry> - </row><row> - <entry> - <para>DECODE(x, encoding)</para> - </entry> - <entry> - <para>Return a blob from the clob with the given encoding. The builtin Java Charset<footnoteref linkend="charset"/> names are valid values for encoding.</para> - </entry> - <entry> - <para>x in a clob, encoding is a string, and returns a blob</para> - </entry> - </row> - <row> - <entry> <para>INITCAP(x)</para> </entry> <entry> @@ -1046,6 +1025,27 @@ </row> <row> <entry> + <para>TO_CHARS(x, encoding)</para> + </entry> + <entry> + <para>Return a clob from the blob with the given encoding. BASE64, HEX, and the builtin Java Charset<footnote id="charset"><para>See the <ulink url="http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charse... JavaDoc</ulink></para></footnote> names are valid values for the encoding.</para> + </entry> + <entry> + <para>x is a blob, encoding is a string, and returns a clob</para> + </entry> + </row><row> + <entry> + <para>TO_BYTES(x, encoding)</para> + </entry> + <entry> + <para>Return a blob from the clob with the given encoding. BASE64, HEX, and the builtin Java Charset<footnoteref linkend="charset"/> names are valid values for the encoding.</para> + </entry> + <entry> + <para>x in a clob, encoding is a string, and returns a blob</para> + </entry> + </row> + <row> + <entry> <para>TRANSLATE(x, y, z)</para> </entry> <entry> Added: trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java (rev 0) +++ trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -0,0 +1,205 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.query.function; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.util.Arrays; + +import org.teiid.core.util.Base64; + +public final class CharsetUtils { + + public static final String HEX_NAME = "HEX"; //$NON-NLS-1$ + static final char[] hex_alphabet = new char[] {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + + public static Charset HEX = new Charset(HEX_NAME, new String[0]) { + + @Override + public CharsetEncoder newEncoder() { + return new FixedEncoder(this, 2, .5f, 1) { + char[] chars = new char[2]; + + @Override + protected void encode(ByteBuffer out) { + this.cb.get(chars); + out.put((byte)(Integer.parseInt(new String(chars), 16) & 0xff)); + } + + }; + } + + @Override + public CharsetDecoder newDecoder() { + return new FixedDecoder(this, 1, 2, 2) { + + @Override + public void decode(CharBuffer out) { + byte b = this.bb.get(); + toHex(out, b); + } + + }; + } + + @Override + public boolean contains(Charset cs) { + return false; + } + }; + + public static void toHex(CharBuffer out, byte b) { + out.put(hex_alphabet[(b & 0xf0) >> 4]); + out.put(hex_alphabet[b & 0x0f]); + } + + public static final String BASE64_NAME = "BASE64"; //$NON-NLS-1$ + public static Charset BASE64 = new Charset(BASE64_NAME, new String[0]) { + @Override + public CharsetEncoder newEncoder() { + return new FixedEncoder(this, 4, .75f, 1) { + + @Override + protected void encode(ByteBuffer out) { + out.put(Base64.decode(cb)); + } + + }; + } + + @Override + public CharsetDecoder newDecoder() { + return new FixedDecoder(this, 3, 1.25f, 3) { + + @Override + public void decode(CharBuffer out) { + if (bb.limit() == bb.array().length) { + out.put(Base64.encodeBytes(bb.array())); + } else { + out.put(Base64.encodeBytes(Arrays.copyOf(bb.array(), bb.limit()))); + } + } + }; + } + + @Override + public boolean contains(Charset cs) { + return false; + } + }; + + public static abstract class FixedEncoder extends CharsetEncoder { + + protected CharBuffer cb; + + protected FixedEncoder(Charset cs, int encodeChars, float averageBytesPerChar, float maxBytesPerChar) { + super(cs, averageBytesPerChar, maxBytesPerChar); + cb = CharBuffer.wrap(new char[encodeChars]); + } + + @Override + protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) { + while (in.hasRemaining()) { + cb.put(in.get()); + if (!cb.hasRemaining()) { + if (!out.hasRemaining()) { + return CoderResult.OVERFLOW; + } + cb.flip(); + encode(out); + cb.clear(); + } + } + return CoderResult.UNDERFLOW; + } + + abstract protected void encode(ByteBuffer out); + + @Override + protected CoderResult implFlush(ByteBuffer out) { + if (cb.position() != 0) { + return CoderResult.unmappableForLength(cb.position()); + } + return super.implFlush(out); + } + + @Override + protected void implReset() { + cb.clear(); + } + + } + + public static abstract class FixedDecoder extends CharsetDecoder { + + protected ByteBuffer bb; + + protected FixedDecoder(Charset cs, int decodeBytes, + float averageCharsPerByte, + float maxCharsPerByte) { + super(cs, averageCharsPerByte, maxCharsPerByte); + this.bb = ByteBuffer.wrap(new byte[decodeBytes]); + } + + @Override + protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) { + while (in.hasRemaining()) { + bb.put(in.get()); + if (!bb.hasRemaining()) { + if (!out.hasRemaining()) { + return CoderResult.OVERFLOW; + } + bb.flip(); + decode(out); + bb.clear(); + } + } + return CoderResult.UNDERFLOW; + } + + protected abstract void decode(CharBuffer out); + + @Override + protected CoderResult implFlush(CharBuffer out) { + if (bb.position() != 0) { + if (!out.hasRemaining()) { + return CoderResult.OVERFLOW; + } + bb.flip(); + decode(out); + bb.clear(); + } + return super.implFlush(out); + } + + @Override + protected void implReset() { + bb.clear(); + } + } + +} \ No newline at end of file Property changes on: trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -25,11 +25,7 @@ import java.io.Serializable; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CoderResult; import java.sql.Blob; import java.sql.Clob; import java.sql.Time; @@ -53,14 +49,15 @@ import org.teiid.core.types.TransformationException; import org.teiid.core.types.InputStreamFactory.BlobInputStreamFactory; import org.teiid.core.types.InputStreamFactory.ClobInputStreamFactory; -import org.teiid.core.util.Base64; import org.teiid.core.util.TimestampWithTimezone; import org.teiid.language.SQLConstants.NonReserved; import org.teiid.query.QueryPlugin; import org.teiid.query.util.CommandContext; import org.teiid.query.util.ErrorMessageKeys; - +/** + * Static method hooks for most of the function library. + */ public final class FunctionMethods { // ================== Function = plus ===================== @@ -1260,102 +1257,29 @@ return TimestampWithTimezone.createTimestamp(value, context.getServerTimeZone(), cal); } - public static Blob decode(ClobType value, String encoding) { - /*if ("HEX".equalsIgnoreCase(encoding)) { - - } - if ("BASE64".equalsIgnoreCase(encoding)) { - - }*/ - ClobInputStreamFactory cisf = new ClobInputStreamFactory(value.getReference()); - cisf.setCharset(Charset.forName(encoding)); - return new BlobType(new BlobImpl(cisf)); - } - - public static Clob encode(BlobType value, String encoding) { - /*if ("HEX".equalsIgnoreCase(encoding)) { - - } - if ("BASE64".equalsIgnoreCase(encoding)) { - - }*/ - BlobInputStreamFactory bisf = new BlobInputStreamFactory(value.getReference()); + public static Clob toChars(BlobType value, String encoding) { + Charset cs = getCharset(encoding); + BlobInputStreamFactory bisf = new BlobInputStreamFactory(value.getReference()); ClobImpl clob = new ClobImpl(bisf, -1); - clob.setEncoding(encoding); + clob.setCharset(cs); return new ClobType(clob); } - public static class Base64Encoder extends CharsetEncoder { - - private CharBuffer cb = CharBuffer.wrap(new char[4]); - - protected Base64Encoder() { - super(Charset.forName("US-ASCII"), .75f, 1); //$NON-NLS-1$ - } - - @Override - protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) { - while (in.hasRemaining()) { - cb.put(in.get()); - if (!cb.hasRemaining()) { - if (!out.hasRemaining()) { - return CoderResult.OVERFLOW; - } - cb.position(0); - out.put(Base64.decode(cb)); - cb.clear(); - } - } - return CoderResult.UNDERFLOW; + public static Blob toBytes(ClobType value, String encoding) { + Charset cs = getCharset(encoding); + ClobInputStreamFactory cisf = new ClobInputStreamFactory(value.getReference()); + cisf.setCharset(cs); + return new BlobType(new BlobImpl(cisf)); + } + + public static Charset getCharset(String encoding) { + if (CharsetUtils.BASE64_NAME.equalsIgnoreCase(encoding)) { + return CharsetUtils.BASE64; } - - @Override - protected CoderResult implFlush(ByteBuffer out) { - if (cb.position() != 0) { - return CoderResult.unmappableForLength(cb.position()); - } - return super.implFlush(out); - } - } + if (CharsetUtils.HEX_NAME.equalsIgnoreCase(encoding)) { + return CharsetUtils.HEX; + } + return Charset.forName(encoding); + } -/* This does not seem to work, since the flush is never called - public static class Base64Decoder extends CharsetDecoder { - - private ByteBuffer bb = ByteBuffer.wrap(new byte[3]); - - protected Base64Decoder() { - super(Charset.forName("US-ASCII"), 1.25f, 3); //$NON-NLS-1$ - } - - @Override - protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) { - while (in.hasRemaining()) { - bb.put(in.get()); - if (!bb.hasRemaining()) { - if (!out.hasRemaining()) { - return CoderResult.OVERFLOW; - } - bb.position(0); - out.put(Base64.encodeBytes(bb.array())); - bb.clear(); - } - } - return CoderResult.UNDERFLOW; - } - - @Override - protected CoderResult implFlush(CharBuffer out) { - if (bb.position() != 0) { - if (!out.hasRemaining()) { - return CoderResult.OVERFLOW; - } - byte[] bytes = Arrays.copyOf(bb.array(), bb.position()); - out.put(Base64.encodeBytes(bytes)); - bb.clear(); - } - return super.implFlush(out); - } - } -*/ - } Modified: trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -144,8 +144,8 @@ addClobFunction("lower", QueryPlugin.Util.getString("SystemSource.LowerClob_result"), "lowerCase", DataTypeManager.DefaultDataTypes.CLOB); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ addClobFunction("upper", QueryPlugin.Util.getString("SystemSource.UpperClob_result"), "upperCase", DataTypeManager.DefaultDataTypes.CLOB); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - addEncodeFunction(); - addDecodeFunction(); + addToCharsFunction(); + addToBytesFunction(); // conversion addConversionFunctions(); @@ -580,18 +580,18 @@ new FunctionParameter("result", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.Insert_result")) ) ); //$NON-NLS-1$ //$NON-NLS-2$ } - private void addEncodeFunction() { + private void addToCharsFunction() { functions.add( - new FunctionMethod("encode", QueryPlugin.Util.getString("SystemSource.encode_desc"), CONVERSION, FUNCTION_CLASS, "encode", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + new FunctionMethod("to_chars", QueryPlugin.Util.getString("SystemSource.encode_desc"), CONVERSION, FUNCTION_CLASS, "toChars", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ new FunctionParameter[] { new FunctionParameter("value", DataTypeManager.DefaultDataTypes.BLOB, QueryPlugin.Util.getString("SystemSource.encode_arg1")), //$NON-NLS-1$ //$NON-NLS-2$ new FunctionParameter("encoding", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.encode_arg2"))}, //$NON-NLS-1$ //$NON-NLS-2$ new FunctionParameter("result", DataTypeManager.DefaultDataTypes.CLOB, QueryPlugin.Util.getString("SystemSource.encode_result")) ) ); //$NON-NLS-1$ //$NON-NLS-2$ } - private void addDecodeFunction() { + private void addToBytesFunction() { functions.add( - new FunctionMethod("decode", QueryPlugin.Util.getString("SystemSource.decode_desc"), CONVERSION, FUNCTION_CLASS, "decode", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + new FunctionMethod("to_bytes", QueryPlugin.Util.getString("SystemSource.decode_desc"), CONVERSION, FUNCTION_CLASS, "toBytes", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ new FunctionParameter[] { new FunctionParameter("value", DataTypeManager.DefaultDataTypes.CLOB, QueryPlugin.Util.getString("SystemSource.decode_arg1")), //$NON-NLS-1$ //$NON-NLS-2$ new FunctionParameter("encoding", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.decode_arg2"))}, //$NON-NLS-1$ //$NON-NLS-2$ Modified: trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -74,6 +74,7 @@ import org.teiid.core.types.XMLType; import org.teiid.core.types.XMLType.Type; import org.teiid.query.eval.Evaluator; +import org.teiid.query.function.CharsetUtils; import org.teiid.query.function.FunctionMethods; import org.teiid.query.processor.xml.XMLUtil; import org.teiid.query.util.CommandContext; @@ -522,7 +523,11 @@ } private static String escapeChar(char chr) { - return "_u00" + Integer.toHexString(chr).toUpperCase() + "_"; //$NON-NLS-1$ //$NON-NLS-2$ + CharBuffer cb = CharBuffer.allocate(7); + cb.append("_u"); //$NON-NLS-1$ + CharsetUtils.toHex(cb, (byte)(chr >> 8)); + CharsetUtils.toHex(cb, (byte)chr); + return cb.append("_").flip().toString(); //$NON-NLS-1$ } } Modified: trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -22,7 +22,6 @@ package org.teiid.query.validator; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -46,6 +45,7 @@ import org.teiid.query.QueryPlugin; import org.teiid.query.eval.Evaluator; import org.teiid.query.function.FunctionLibrary; +import org.teiid.query.function.FunctionMethods; import org.teiid.query.function.source.XMLSystemFunctions; import org.teiid.query.metadata.SupportConstants; import org.teiid.query.resolver.util.ResolverUtil; @@ -348,9 +348,9 @@ handleValidationError(QueryPlugin.Util.getString("QueryResolver.invalid_xpath", e.getMessage()), obj); //$NON-NLS-1$ } } - } else if(obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.ENCODE) || obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.DECODE)) { + } else if(obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.TO_BYTES) || obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.TO_CHARS)) { try { - Charset.forName((String)((Constant)obj.getArg(1)).getValue()); + FunctionMethods.getCharset((String)((Constant)obj.getArg(1)).getValue()); } catch (IllegalArgumentException e) { handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.invalid_encoding", obj.getArg(1)), obj); //$NON-NLS-1$ } Modified: trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj =================================================================== --- trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj 2010-06-18 01:43:01 UTC (rev 2243) @@ -3229,8 +3229,8 @@ <RPAREN> ) | - LOOKAHEAD(<ID>, {matchesAny(getToken(1).image, "encode", "decode") != null}) ( - funcName = nonReserved("ENCODE", "DECODE") + LOOKAHEAD(<ID>, {matchesAny(getToken(1).image, "to_chars", "to_bytes") != null}) ( + funcName = nonReserved("TO_CHARS", "TO_BYTES") <LPAREN> expression = expression(info) { Modified: trunk/engine/src/main/resources/org/teiid/query/i18n.properties =================================================================== --- trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2010-06-18 01:43:01 UTC (rev 2243) @@ -811,14 +811,14 @@ SystemSource.coalesce_description=Returns the first non-null parameter SystemSource.coalesce_param1=parameter SystemSource.coalesce_result=The first non-null parameter -SystemSource.encode_desc=Encodes the given Blob to a Clob +SystemSource.encode_desc=Converts a Blob to the appropriate chars SystemSource.encode_arg1=Blob -SystemSource.encode_arg2=The encoding -SystemSource.encode_result=The Clob -SystemSource.decode_desc=Decodes a Clob to a Blob +SystemSource.encode_arg2=char representation Clob +SystemSource.encode_result=Clob char rep +SystemSource.decode_desc=Converts a Clob to the appropriate bytes SystemSource.decode_arg1=Clob SystemSource.decode_arg2=The encoding -SystemSource.decode_result=The Blob +SystemSource.decode_result=byte representation Blob TempMetadataAdapter.Element_____{0}_____not_found._1=Element ''{0}'' not found. TempMetadataAdapter.Group_____{0}_____not_found._1=Group ''{0}'' not found. ExpressionEvaluator.Must_push=Function {0} is marked in the function metadata as a function that must be evaluated at the source. Modified: trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -52,6 +52,7 @@ import org.teiid.core.types.NullType; import org.teiid.core.types.XMLType; import org.teiid.core.types.DataTypeManager.DefaultDataClasses; +import org.teiid.core.util.Base64; import org.teiid.core.util.ObjectConverterUtil; import org.teiid.core.util.TimestampWithTimezone; import org.teiid.language.SQLConstants.NonReserved; @@ -1369,14 +1370,31 @@ } @Test public void testEncode() throws Exception { - Clob result = (Clob)helpInvokeMethod("encode", new Class[] {DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new SerialBlob("hello world".getBytes("ASCII"))), "ASCII" }, null); //$NON-NLS-1$ + Clob result = (Clob)helpInvokeMethod("to_chars", new Class[] {DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new SerialBlob("hello world".getBytes("ASCII"))), "ASCII" }, null); //$NON-NLS-1$ String string = result.getSubString(1, (int)result.length()); assertEquals("hello world", string); } @Test public void testDecode() throws Exception { - Blob result = (Blob)helpInvokeMethod("decode", new Class[] {DefaultDataClasses.CLOB, DefaultDataClasses.STRING}, new Object[] { new ClobType(new SerialClob("hello world".toCharArray())), "UTF32" }, null); //$NON-NLS-1$ + Blob result = (Blob)helpInvokeMethod("to_bytes", new Class[] {DefaultDataClasses.CLOB, DefaultDataClasses.STRING}, new Object[] { new ClobType(new SerialClob("hello world".toCharArray())), "UTF32" }, null); //$NON-NLS-1$ assertEquals(44, result.length()); //4 bytes / char } + @Test public void testEncode1() throws Exception { + Clob result = (Clob)helpInvokeMethod("to_chars", new Class[] {DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new SerialBlob("hello world".getBytes("ASCII"))), "BASE64" }, null); //$NON-NLS-1$ + String string = result.getSubString(1, (int)result.length()); + assertEquals("hello world", new String(Base64.decode(string), "ASCII")); + } + + @Test public void testEncode2() throws Exception { + Clob result = (Clob)helpInvokeMethod("to_chars", new Class[] {DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new SerialBlob("hello world".getBytes("ASCII"))), "HEX" }, null); //$NON-NLS-1$ + String string = result.getSubString(1, (int)result.length()); + assertEquals("68656C6C6F20776F726C64", string); + } + + @Test public void testDecode2() throws Exception { + Blob result = (Blob)helpInvokeMethod("to_bytes", new Class[] {DefaultDataClasses.CLOB, DefaultDataClasses.STRING}, new Object[] { new ClobType(new SerialClob("68656C6C6F20776F726C64".toCharArray())), "HEX" }, null); //$NON-NLS-1$ + assertEquals("hello world", new String(ObjectConverterUtil.convertToCharArray(result.getBinaryStream(), -1, "ASCII"))); + } + } Modified: trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -209,6 +209,10 @@ assertEquals("a_u005F_x", XMLSystemFunctions.escapeName("a_x", true)); } + @Test public void testNameEscaping2() throws Exception { + assertEquals("_u000A_", XMLSystemFunctions.escapeName(new String(new char[] {10}), true)); + } + @BeforeClass static public void setUpOnce() { TimeZone.setDefault(TimeZone.getTimeZone("GMT-6:00")); } Modified: trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java 2010-06-17 20:14:18 UTC (rev 2242) +++ trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java 2010-06-18 01:43:01 UTC (rev 2243) @@ -2036,7 +2036,7 @@ } @Test public void testDecode() throws Exception { - helpValidate("select decode(e1, '?') from pm1.g1", new String[] {"decode(e1, '?')"}, FakeMetadataFactory.example1Cached()); + helpValidate("select to_bytes(e1, '?') from pm1.g1", new String[] {"to_bytes(e1, '?')"}, FakeMetadataFactory.example1Cached()); } }