8:43 p.m.
Author: shawkins
Date: 2010-06-17 21:43:01 -0400 (Thu, 17 Jun 2010)
New Revision: 2243
Added:
trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java
trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java
Removed:
trunk/documentation/quick-start-example/src/main/docbook/en-US/images/EmbeddedDesignerExampleVideo.swf
Modified:
trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java
trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java
trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java
trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java
trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java
trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java
trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml
trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java
trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java
trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java
trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java
trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj
trunk/engine/src/main/resources/org/teiid/query/i18n.properties
trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java
trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java
trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java
Log:
TEIID-1021 refining support for clob<->blob conversions. changes encode/decode to
to_bytes/to_chars to cover both binary and character encodings.
Modified: trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java
===================================================================
--- trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/api/src/main/java/org/teiid/translator/SourceSystemFunctions.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -40,8 +40,6 @@
public static final String ASCII = "ascii"; //$NON-NLS-1$
public static final String CHAR = "char"; //$NON-NLS-1$
public static final String CONCAT = "concat"; //$NON-NLS-1$
- public static final String DECODE = "decode"; //$NON-NLS-1$
- public static final String ENCODE = "encode"; //$NON-NLS-1$
//public static final String DIFFERENCE = "difference";
public static final String INITCAP = "initcap"; //$NON-NLS-1$
public static final String INSERT = "insert"; //$NON-NLS-1$
@@ -58,6 +56,8 @@
public static final String RTRIM = "rtrim"; //$NON-NLS-1$
//public static final String SOUNDEX = "soundex";
public static final String SUBSTRING = "substring"; //$NON-NLS-1$
+ public static final String TO_BYTES = "to_bytes"; //$NON-NLS-1$
+ public static final String TO_CHARS = "to_chars"; //$NON-NLS-1$
public static final String UCASE = "ucase"; //$NON-NLS-1$
//numeric
Modified: trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2010-06-17 20:14:18
UTC (rev 2242)
+++ trunk/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2010-06-18 01:43:01
UTC (rev 2243)
@@ -1,3 +1,25 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
package org.teiid.core.types;
import java.io.Externalizable;
@@ -3,5 +25,4 @@
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -11,13 +32,13 @@
import java.sql.SQLException;
import org.teiid.core.types.InputStreamFactory.StreamFactoryReference;
+import org.teiid.core.util.InputStreamReader;
-
public class BaseLob implements Externalizable, StreamFactoryReference {
private static final long serialVersionUID = -1586959324208959519L;
private InputStreamFactory streamFactory;
- private String encoding;
+ private Charset charset;
public BaseLob() {
@@ -38,14 +59,22 @@
return streamFactory;
}
- public String getEncoding() {
- return encoding;
+ public void setEncoding(String encoding) {
+ if (encoding != null) {
+ this.charset = Charset.forName(encoding);
+ } else {
+ this.charset = null;
+ }
}
- public void setEncoding(String encoding) {
- this.encoding = encoding;
+ public Charset getCharset() {
+ return charset;
}
+ public void setCharset(Charset charset) {
+ this.charset = charset;
+ }
+
public void free() throws SQLException {
if (this.streamFactory != null) {
try {
@@ -70,11 +99,11 @@
ex.initCause(e);
throw ex;
}
- String enc = getEncoding();
- if (enc == null) {
- enc = Streamable.ENCODING;
+ Charset cs = getCharset();
+ if (cs == null) {
+ cs = Streamable.CHARSET;
}
- return new InputStreamReader(getBinaryStream(), Charset.forName(enc));
+ return new InputStreamReader(getBinaryStream(), cs.newDecoder());
}
public InputStream getBinaryStream() throws SQLException {
Modified: trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -84,19 +84,19 @@
}
@Override
- public String getEncoding() {
- String enc = super.getEncoding();
- if (enc != null) {
- return enc;
+ public Charset getCharset() {
+ Charset cs = super.getCharset();
+ if (cs != null) {
+ return cs;
}
try {
- enc = XMLType.getEncoding(this.getBinaryStream());
+ String enc = XMLType.getEncoding(this.getBinaryStream());
if (enc != null) {
setEncoding(enc);
}
} catch (SQLException e) {
}
- return Streamable.ENCODING;
+ return Streamable.CHARSET;
}
@SuppressWarnings("unchecked")
Modified: trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/common-core/src/main/java/org/teiid/core/types/Streamable.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -26,6 +26,7 @@
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.nio.charset.Charset;
import org.teiid.core.CorePlugin;
@@ -43,6 +44,7 @@
private static final long serialVersionUID = -8252488562134729374L;
public static final String ENCODING = "UTF-8"; //$NON-NLS-1$
+ public static final Charset CHARSET = Charset.forName(ENCODING);
public static final String FORCE_STREAMING = "FORCE_STREAMING";
//$NON-NLS-1$
public static final int STREAMING_BATCH_SIZE_IN_BYTES = 102400; // 100K
Modified: trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java 2010-06-17 20:14:18
UTC (rev 2242)
+++ trunk/common-core/src/main/java/org/teiid/core/types/XMLType.java 2010-06-18 01:43:01
UTC (rev 2243)
@@ -29,6 +29,7 @@
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
+import java.nio.charset.Charset;
import java.sql.SQLException;
import java.sql.SQLXML;
@@ -147,7 +148,10 @@
xml = type.reference;
}
if (xml instanceof SQLXMLImpl) {
- return ((SQLXMLImpl)xml).getEncoding();
+ Charset cs = ((SQLXMLImpl)xml).getCharset();
+ if (cs != null) {
+ return cs.displayName();
+ }
}
return getEncoding(xml.getBinaryStream());
} catch (SQLException e) {
Added: trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java
(rev 0)
+++ trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -0,0 +1,104 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.core.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
+
+import org.teiid.core.types.DataTypeManager;
+
+/**
+ * Replacement for the standard {@link java.io.InputStreamReader},
+ * which suffers from a <a
href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4744247&quo...
in sun.nio.cs.StreamDecoder
+ */
+public class InputStreamReader extends Reader {
+
+ private CharsetDecoder cd;
+ private ReadableByteChannel rbc;
+ private ByteBuffer bb;
+ private CharBuffer cb;
+ private boolean done;
+
+ public InputStreamReader(InputStream in, CharsetDecoder cd) {
+ this(in, cd, DataTypeManager.MAX_LOB_MEMORY_BYTES);
+ }
+
+ public InputStreamReader(InputStream in, CharsetDecoder cd, int bufferSize) {
+ this.cd = cd;
+ this.rbc = Channels.newChannel(in);
+ this.bb = ByteBuffer.allocate(bufferSize);
+ this.cb = CharBuffer.allocate((int)(bufferSize * (double)cd.maxCharsPerByte()));
+ this.cb.limit(0);
+ }
+
+ @Override
+ public void close() throws IOException {
+ rbc.close();
+ cd.reset();
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ if ((off < 0) || (off > cbuf.length) || (len < 0) ||
+ ((off + len) > cbuf.length) || ((off + len) < 0)) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+ while (!done && !cb.hasRemaining()) {
+ int read = 0;
+ while ((read = rbc.read(bb)) == 0) {
+ //blocking read
+ }
+ bb.flip();
+ cb.clear();
+ CoderResult cr = cd.decode(bb, cb, read == -1);
+ if (!cr.isUnderflow()) {
+ cr.throwException();
+ }
+ if (read == -1) {
+ cr = cd.flush(cb);
+ if (!cr.isUnderflow()) {
+ cr.throwException();
+ }
+ done = true;
+ }
+ bb.clear();
+ cb.flip();
+ }
+ len = Math.min(len, cb.remaining());
+ if (len == 0 && done) {
+ return -1;
+ }
+ cb.get(cbuf, off, len);
+ return len;
+ }
+
+}
\ No newline at end of file
Property changes on:
trunk/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Modified: trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/common-core/src/main/java/org/teiid/core/util/ReaderInputStream.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -47,15 +47,15 @@
private char[] charBuffer;
private AccessibleByteArrayOutputStream out = new AccessibleByteArrayOutputStream();
private boolean hasMore = true;
- private int pos = 0;
+ private int pos;
- public ReaderInputStream(Reader reader, Charset charSet) {
- this(reader, charSet, DEFAULT_BUFFER_SIZE);
+ public ReaderInputStream(Reader reader, Charset charset) {
+ this(reader, charset, DEFAULT_BUFFER_SIZE);
}
- public ReaderInputStream(Reader reader, Charset charSet, int bufferSize) {
+ public ReaderInputStream(Reader reader, Charset charset, int bufferSize) {
this.reader = reader;
- this.writer = new OutputStreamWriter(out, charSet);
+ this.writer = new OutputStreamWriter(out, charset);
this.charBuffer = new char[bufferSize];
}
Deleted:
trunk/documentation/quick-start-example/src/main/docbook/en-US/images/EmbeddedDesignerExampleVideo.swf
===================================================================
(Binary files differ)
Modified:
trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml
===================================================================
---
trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml 2010-06-17
20:14:18 UTC (rev 2242)
+++
trunk/documentation/reference/src/main/docbook/en-US/content/scalar_functions.xml 2010-06-18
01:43:01 UTC (rev 2243)
@@ -801,27 +801,6 @@
</row>
<row>
<entry>
- <para>ENCODE(x, encoding)</para>
- </entry>
- <entry>
- <para>Return a clob from the blob with the given encoding. The
builtin Java Charset<footnote id="charset"><para>See the <ulink
url="http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charse...
JavaDoc</ulink></para></footnote> names are valid values for
encoding.</para>
- </entry>
- <entry>
- <para>x is a blob, encoding is a string, and returns a
clob</para>
- </entry>
- </row><row>
- <entry>
- <para>DECODE(x, encoding)</para>
- </entry>
- <entry>
- <para>Return a blob from the clob with the given encoding. The
builtin Java Charset<footnoteref linkend="charset"/> names are valid
values for encoding.</para>
- </entry>
- <entry>
- <para>x in a clob, encoding is a string, and returns a
blob</para>
- </entry>
- </row>
- <row>
- <entry>
<para>INITCAP(x)</para>
</entry>
<entry>
@@ -1046,6 +1025,27 @@
</row>
<row>
<entry>
+ <para>TO_CHARS(x, encoding)</para>
+ </entry>
+ <entry>
+ <para>Return a clob from the blob with the given encoding. BASE64,
HEX, and the builtin Java Charset<footnote id="charset"><para>See
the <ulink
url="http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charse...
JavaDoc</ulink></para></footnote> names are valid values for the
encoding.</para>
+ </entry>
+ <entry>
+ <para>x is a blob, encoding is a string, and returns a
clob</para>
+ </entry>
+ </row><row>
+ <entry>
+ <para>TO_BYTES(x, encoding)</para>
+ </entry>
+ <entry>
+ <para>Return a blob from the clob with the given encoding. BASE64,
HEX, and the builtin Java Charset<footnoteref linkend="charset"/> names
are valid values for the encoding.</para>
+ </entry>
+ <entry>
+ <para>x in a clob, encoding is a string, and returns a
blob</para>
+ </entry>
+ </row>
+ <row>
+ <entry>
<para>TRANSLATE(x, y, z)</para>
</entry>
<entry>
Added: trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java
(rev 0)
+++ trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -0,0 +1,205 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.query.function;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.util.Arrays;
+
+import org.teiid.core.util.Base64;
+
+public final class CharsetUtils {
+
+ public static final String HEX_NAME = "HEX"; //$NON-NLS-1$
+ static final char[] hex_alphabet = new char[] {'0', '1', '2',
'3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F'};
+
+ public static Charset HEX = new Charset(HEX_NAME, new String[0]) {
+
+ @Override
+ public CharsetEncoder newEncoder() {
+ return new FixedEncoder(this, 2, .5f, 1) {
+ char[] chars = new char[2];
+
+ @Override
+ protected void encode(ByteBuffer out) {
+ this.cb.get(chars);
+ out.put((byte)(Integer.parseInt(new String(chars), 16) & 0xff));
+ }
+
+ };
+ }
+
+ @Override
+ public CharsetDecoder newDecoder() {
+ return new FixedDecoder(this, 1, 2, 2) {
+
+ @Override
+ public void decode(CharBuffer out) {
+ byte b = this.bb.get();
+ toHex(out, b);
+ }
+
+ };
+ }
+
+ @Override
+ public boolean contains(Charset cs) {
+ return false;
+ }
+ };
+
+ public static void toHex(CharBuffer out, byte b) {
+ out.put(hex_alphabet[(b & 0xf0) >> 4]);
+ out.put(hex_alphabet[b & 0x0f]);
+ }
+
+ public static final String BASE64_NAME = "BASE64"; //$NON-NLS-1$
+ public static Charset BASE64 = new Charset(BASE64_NAME, new String[0]) {
+ @Override
+ public CharsetEncoder newEncoder() {
+ return new FixedEncoder(this, 4, .75f, 1) {
+
+ @Override
+ protected void encode(ByteBuffer out) {
+ out.put(Base64.decode(cb));
+ }
+
+ };
+ }
+
+ @Override
+ public CharsetDecoder newDecoder() {
+ return new FixedDecoder(this, 3, 1.25f, 3) {
+
+ @Override
+ public void decode(CharBuffer out) {
+ if (bb.limit() == bb.array().length) {
+ out.put(Base64.encodeBytes(bb.array()));
+ } else {
+ out.put(Base64.encodeBytes(Arrays.copyOf(bb.array(), bb.limit())));
+ }
+ }
+ };
+ }
+
+ @Override
+ public boolean contains(Charset cs) {
+ return false;
+ }
+ };
+
+ public static abstract class FixedEncoder extends CharsetEncoder {
+
+ protected CharBuffer cb;
+
+ protected FixedEncoder(Charset cs, int encodeChars, float averageBytesPerChar, float
maxBytesPerChar) {
+ super(cs, averageBytesPerChar, maxBytesPerChar);
+ cb = CharBuffer.wrap(new char[encodeChars]);
+ }
+
+ @Override
+ protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) {
+ while (in.hasRemaining()) {
+ cb.put(in.get());
+ if (!cb.hasRemaining()) {
+ if (!out.hasRemaining()) {
+ return CoderResult.OVERFLOW;
+ }
+ cb.flip();
+ encode(out);
+ cb.clear();
+ }
+ }
+ return CoderResult.UNDERFLOW;
+ }
+
+ abstract protected void encode(ByteBuffer out);
+
+ @Override
+ protected CoderResult implFlush(ByteBuffer out) {
+ if (cb.position() != 0) {
+ return CoderResult.unmappableForLength(cb.position());
+ }
+ return super.implFlush(out);
+ }
+
+ @Override
+ protected void implReset() {
+ cb.clear();
+ }
+
+ }
+
+ public static abstract class FixedDecoder extends CharsetDecoder {
+
+ protected ByteBuffer bb;
+
+ protected FixedDecoder(Charset cs, int decodeBytes,
+ float averageCharsPerByte,
+ float maxCharsPerByte) {
+ super(cs, averageCharsPerByte, maxCharsPerByte);
+ this.bb = ByteBuffer.wrap(new byte[decodeBytes]);
+ }
+
+ @Override
+ protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
+ while (in.hasRemaining()) {
+ bb.put(in.get());
+ if (!bb.hasRemaining()) {
+ if (!out.hasRemaining()) {
+ return CoderResult.OVERFLOW;
+ }
+ bb.flip();
+ decode(out);
+ bb.clear();
+ }
+ }
+ return CoderResult.UNDERFLOW;
+ }
+
+ protected abstract void decode(CharBuffer out);
+
+ @Override
+ protected CoderResult implFlush(CharBuffer out) {
+ if (bb.position() != 0) {
+ if (!out.hasRemaining()) {
+ return CoderResult.OVERFLOW;
+ }
+ bb.flip();
+ decode(out);
+ bb.clear();
+ }
+ return super.implFlush(out);
+ }
+
+ @Override
+ protected void implReset() {
+ bb.clear();
+ }
+ }
+
+}
\ No newline at end of file
Property changes on:
trunk/engine/src/main/java/org/teiid/query/function/CharsetUtils.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Modified: trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/engine/src/main/java/org/teiid/query/function/FunctionMethods.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -25,11 +25,7 @@
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
import java.nio.charset.Charset;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CoderResult;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.Time;
@@ -53,14 +49,15 @@
import org.teiid.core.types.TransformationException;
import org.teiid.core.types.InputStreamFactory.BlobInputStreamFactory;
import org.teiid.core.types.InputStreamFactory.ClobInputStreamFactory;
-import org.teiid.core.util.Base64;
import org.teiid.core.util.TimestampWithTimezone;
import org.teiid.language.SQLConstants.NonReserved;
import org.teiid.query.QueryPlugin;
import org.teiid.query.util.CommandContext;
import org.teiid.query.util.ErrorMessageKeys;
-
+/**
+ * Static method hooks for most of the function library.
+ */
public final class FunctionMethods {
// ================== Function = plus =====================
@@ -1260,102 +1257,29 @@
return TimestampWithTimezone.createTimestamp(value, context.getServerTimeZone(),
cal);
}
- public static Blob decode(ClobType value, String encoding) {
- /*if ("HEX".equalsIgnoreCase(encoding)) {
-
- }
- if ("BASE64".equalsIgnoreCase(encoding)) {
-
- }*/
- ClobInputStreamFactory cisf = new ClobInputStreamFactory(value.getReference());
- cisf.setCharset(Charset.forName(encoding));
- return new BlobType(new BlobImpl(cisf));
- }
-
- public static Clob encode(BlobType value, String encoding) {
- /*if ("HEX".equalsIgnoreCase(encoding)) {
-
- }
- if ("BASE64".equalsIgnoreCase(encoding)) {
-
- }*/
- BlobInputStreamFactory bisf = new BlobInputStreamFactory(value.getReference());
+ public static Clob toChars(BlobType value, String encoding) {
+ Charset cs = getCharset(encoding);
+ BlobInputStreamFactory bisf = new BlobInputStreamFactory(value.getReference());
ClobImpl clob = new ClobImpl(bisf, -1);
- clob.setEncoding(encoding);
+ clob.setCharset(cs);
return new ClobType(clob);
}
- public static class Base64Encoder extends CharsetEncoder {
-
- private CharBuffer cb = CharBuffer.wrap(new char[4]);
-
- protected Base64Encoder() {
- super(Charset.forName("US-ASCII"), .75f, 1); //$NON-NLS-1$
- }
-
- @Override
- protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) {
- while (in.hasRemaining()) {
- cb.put(in.get());
- if (!cb.hasRemaining()) {
- if (!out.hasRemaining()) {
- return CoderResult.OVERFLOW;
- }
- cb.position(0);
- out.put(Base64.decode(cb));
- cb.clear();
- }
- }
- return CoderResult.UNDERFLOW;
+ public static Blob toBytes(ClobType value, String encoding) {
+ Charset cs = getCharset(encoding);
+ ClobInputStreamFactory cisf = new ClobInputStreamFactory(value.getReference());
+ cisf.setCharset(cs);
+ return new BlobType(new BlobImpl(cisf));
+ }
+
+ public static Charset getCharset(String encoding) {
+ if (CharsetUtils.BASE64_NAME.equalsIgnoreCase(encoding)) {
+ return CharsetUtils.BASE64;
}
-
- @Override
- protected CoderResult implFlush(ByteBuffer out) {
- if (cb.position() != 0) {
- return CoderResult.unmappableForLength(cb.position());
- }
- return super.implFlush(out);
- }
- }
+ if (CharsetUtils.HEX_NAME.equalsIgnoreCase(encoding)) {
+ return CharsetUtils.HEX;
+ }
+ return Charset.forName(encoding);
+ }
-/* This does not seem to work, since the flush is never called
- public static class Base64Decoder extends CharsetDecoder {
-
- private ByteBuffer bb = ByteBuffer.wrap(new byte[3]);
-
- protected Base64Decoder() {
- super(Charset.forName("US-ASCII"), 1.25f, 3); //$NON-NLS-1$
- }
-
- @Override
- protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
- while (in.hasRemaining()) {
- bb.put(in.get());
- if (!bb.hasRemaining()) {
- if (!out.hasRemaining()) {
- return CoderResult.OVERFLOW;
- }
- bb.position(0);
- out.put(Base64.encodeBytes(bb.array()));
- bb.clear();
- }
- }
- return CoderResult.UNDERFLOW;
- }
-
- @Override
- protected CoderResult implFlush(CharBuffer out) {
- if (bb.position() != 0) {
- if (!out.hasRemaining()) {
- return CoderResult.OVERFLOW;
- }
- byte[] bytes = Arrays.copyOf(bb.array(), bb.position());
- out.put(Base64.encodeBytes(bytes));
- bb.clear();
- }
- return super.implFlush(out);
- }
- }
-*/
-
}
Modified: trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java
===================================================================
---
trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java 2010-06-17
20:14:18 UTC (rev 2242)
+++
trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -144,8 +144,8 @@
addClobFunction("lower",
QueryPlugin.Util.getString("SystemSource.LowerClob_result"),
"lowerCase", DataTypeManager.DefaultDataTypes.CLOB); //$NON-NLS-1$ //$NON-NLS-2$
//$NON-NLS-3$
addClobFunction("upper",
QueryPlugin.Util.getString("SystemSource.UpperClob_result"),
"upperCase", DataTypeManager.DefaultDataTypes.CLOB); //$NON-NLS-1$ //$NON-NLS-2$
//$NON-NLS-3$
- addEncodeFunction();
- addDecodeFunction();
+ addToCharsFunction();
+ addToBytesFunction();
// conversion
addConversionFunctions();
@@ -580,18 +580,18 @@
new FunctionParameter("result", DataTypeManager.DefaultDataTypes.STRING,
QueryPlugin.Util.getString("SystemSource.Insert_result")) ) );
//$NON-NLS-1$ //$NON-NLS-2$
}
- private void addEncodeFunction() {
+ private void addToCharsFunction() {
functions.add(
- new FunctionMethod("encode",
QueryPlugin.Util.getString("SystemSource.encode_desc"), CONVERSION,
FUNCTION_CLASS, "encode", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ new FunctionMethod("to_chars",
QueryPlugin.Util.getString("SystemSource.encode_desc"), CONVERSION,
FUNCTION_CLASS, "toChars", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
new FunctionParameter[] {
new FunctionParameter("value", DataTypeManager.DefaultDataTypes.BLOB,
QueryPlugin.Util.getString("SystemSource.encode_arg1")), //$NON-NLS-1$
//$NON-NLS-2$
new FunctionParameter("encoding", DataTypeManager.DefaultDataTypes.STRING,
QueryPlugin.Util.getString("SystemSource.encode_arg2"))}, //$NON-NLS-1$
//$NON-NLS-2$
new FunctionParameter("result", DataTypeManager.DefaultDataTypes.CLOB,
QueryPlugin.Util.getString("SystemSource.encode_result")) ) );
//$NON-NLS-1$ //$NON-NLS-2$
}
- private void addDecodeFunction() {
+ private void addToBytesFunction() {
functions.add(
- new FunctionMethod("decode",
QueryPlugin.Util.getString("SystemSource.decode_desc"), CONVERSION,
FUNCTION_CLASS, "decode", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ new FunctionMethod("to_bytes",
QueryPlugin.Util.getString("SystemSource.decode_desc"), CONVERSION,
FUNCTION_CLASS, "toBytes", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
new FunctionParameter[] {
new FunctionParameter("value", DataTypeManager.DefaultDataTypes.CLOB,
QueryPlugin.Util.getString("SystemSource.decode_arg1")), //$NON-NLS-1$
//$NON-NLS-2$
new FunctionParameter("encoding", DataTypeManager.DefaultDataTypes.STRING,
QueryPlugin.Util.getString("SystemSource.decode_arg2"))}, //$NON-NLS-1$
//$NON-NLS-2$
Modified:
trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java
===================================================================
---
trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java 2010-06-17
20:14:18 UTC (rev 2242)
+++
trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -74,6 +74,7 @@
import org.teiid.core.types.XMLType;
import org.teiid.core.types.XMLType.Type;
import org.teiid.query.eval.Evaluator;
+import org.teiid.query.function.CharsetUtils;
import org.teiid.query.function.FunctionMethods;
import org.teiid.query.processor.xml.XMLUtil;
import org.teiid.query.util.CommandContext;
@@ -522,7 +523,11 @@
}
private static String escapeChar(char chr) {
- return "_u00" + Integer.toHexString(chr).toUpperCase() + "_";
//$NON-NLS-1$ //$NON-NLS-2$
+ CharBuffer cb = CharBuffer.allocate(7);
+ cb.append("_u"); //$NON-NLS-1$
+ CharsetUtils.toHex(cb, (byte)(chr >> 8));
+ CharsetUtils.toHex(cb, (byte)chr);
+ return cb.append("_").flip().toString(); //$NON-NLS-1$
}
}
Modified: trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/engine/src/main/java/org/teiid/query/validator/ValidationVisitor.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -22,7 +22,6 @@
package org.teiid.query.validator;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -46,6 +45,7 @@
import org.teiid.query.QueryPlugin;
import org.teiid.query.eval.Evaluator;
import org.teiid.query.function.FunctionLibrary;
+import org.teiid.query.function.FunctionMethods;
import org.teiid.query.function.source.XMLSystemFunctions;
import org.teiid.query.metadata.SupportConstants;
import org.teiid.query.resolver.util.ResolverUtil;
@@ -348,9 +348,9 @@
handleValidationError(QueryPlugin.Util.getString("QueryResolver.invalid_xpath",
e.getMessage()), obj); //$NON-NLS-1$
}
}
- } else
if(obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.ENCODE) ||
obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.DECODE)) {
+ } else
if(obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.TO_BYTES)
|| obj.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.TO_CHARS))
{
try {
- Charset.forName((String)((Constant)obj.getArg(1)).getValue());
+ FunctionMethods.getCharset((String)((Constant)obj.getArg(1)).getValue());
} catch (IllegalArgumentException e) {
handleValidationError(QueryPlugin.Util.getString("ValidationVisitor.invalid_encoding",
obj.getArg(1)), obj); //$NON-NLS-1$
}
Modified: trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj
===================================================================
--- trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj 2010-06-17 20:14:18
UTC (rev 2242)
+++ trunk/engine/src/main/javacc/org/teiid/query/parser/SQLParser.jj 2010-06-18 01:43:01
UTC (rev 2243)
@@ -3229,8 +3229,8 @@
<RPAREN>
)
|
- LOOKAHEAD(<ID>, {matchesAny(getToken(1).image, "encode",
"decode") != null}) (
- funcName = nonReserved("ENCODE", "DECODE")
+ LOOKAHEAD(<ID>, {matchesAny(getToken(1).image, "to_chars",
"to_bytes") != null}) (
+ funcName = nonReserved("TO_CHARS", "TO_BYTES")
<LPAREN>
expression = expression(info)
{
Modified: trunk/engine/src/main/resources/org/teiid/query/i18n.properties
===================================================================
--- trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2010-06-17 20:14:18
UTC (rev 2242)
+++ trunk/engine/src/main/resources/org/teiid/query/i18n.properties 2010-06-18 01:43:01
UTC (rev 2243)
@@ -811,14 +811,14 @@
SystemSource.coalesce_description=Returns the first non-null parameter
SystemSource.coalesce_param1=parameter
SystemSource.coalesce_result=The first non-null parameter
-SystemSource.encode_desc=Encodes the given Blob to a Clob
+SystemSource.encode_desc=Converts a Blob to the appropriate chars
SystemSource.encode_arg1=Blob
-SystemSource.encode_arg2=The encoding
-SystemSource.encode_result=The Clob
-SystemSource.decode_desc=Decodes a Clob to a Blob
+SystemSource.encode_arg2=char representation Clob
+SystemSource.encode_result=Clob char rep
+SystemSource.decode_desc=Converts a Clob to the appropriate bytes
SystemSource.decode_arg1=Clob
SystemSource.decode_arg2=The encoding
-SystemSource.decode_result=The Blob
+SystemSource.decode_result=byte representation Blob
TempMetadataAdapter.Element_____{0}_____not_found._1=Element ''{0}'' not
found.
TempMetadataAdapter.Group_____{0}_____not_found._1=Group ''{0}'' not
found.
ExpressionEvaluator.Must_push=Function {0} is marked in the function metadata as a
function that must be evaluated at the source.
Modified: trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java
===================================================================
---
trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java 2010-06-17
20:14:18 UTC (rev 2242)
+++
trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -52,6 +52,7 @@
import org.teiid.core.types.NullType;
import org.teiid.core.types.XMLType;
import org.teiid.core.types.DataTypeManager.DefaultDataClasses;
+import org.teiid.core.util.Base64;
import org.teiid.core.util.ObjectConverterUtil;
import org.teiid.core.util.TimestampWithTimezone;
import org.teiid.language.SQLConstants.NonReserved;
@@ -1369,14 +1370,31 @@
}
@Test public void testEncode() throws Exception {
- Clob result = (Clob)helpInvokeMethod("encode", new Class[]
{DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new
SerialBlob("hello world".getBytes("ASCII"))), "ASCII" },
null); //$NON-NLS-1$
+ Clob result = (Clob)helpInvokeMethod("to_chars", new Class[]
{DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new
SerialBlob("hello world".getBytes("ASCII"))), "ASCII" },
null); //$NON-NLS-1$
String string = result.getSubString(1, (int)result.length());
assertEquals("hello world", string);
}
@Test public void testDecode() throws Exception {
- Blob result = (Blob)helpInvokeMethod("decode", new Class[]
{DefaultDataClasses.CLOB, DefaultDataClasses.STRING}, new Object[] { new ClobType(new
SerialClob("hello world".toCharArray())), "UTF32" }, null);
//$NON-NLS-1$
+ Blob result = (Blob)helpInvokeMethod("to_bytes", new Class[]
{DefaultDataClasses.CLOB, DefaultDataClasses.STRING}, new Object[] { new ClobType(new
SerialClob("hello world".toCharArray())), "UTF32" }, null);
//$NON-NLS-1$
assertEquals(44, result.length()); //4 bytes / char
}
+ @Test public void testEncode1() throws Exception {
+ Clob result = (Clob)helpInvokeMethod("to_chars", new Class[]
{DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new
SerialBlob("hello world".getBytes("ASCII"))), "BASE64" },
null); //$NON-NLS-1$
+ String string = result.getSubString(1, (int)result.length());
+ assertEquals("hello world", new String(Base64.decode(string),
"ASCII"));
+ }
+
+ @Test public void testEncode2() throws Exception {
+ Clob result = (Clob)helpInvokeMethod("to_chars", new Class[]
{DefaultDataClasses.BLOB, DefaultDataClasses.STRING}, new Object[] { new BlobType(new
SerialBlob("hello world".getBytes("ASCII"))), "HEX" },
null); //$NON-NLS-1$
+ String string = result.getSubString(1, (int)result.length());
+ assertEquals("68656C6C6F20776F726C64", string);
+ }
+
+ @Test public void testDecode2() throws Exception {
+ Blob result = (Blob)helpInvokeMethod("to_bytes", new Class[]
{DefaultDataClasses.CLOB, DefaultDataClasses.STRING}, new Object[] { new ClobType(new
SerialClob("68656C6C6F20776F726C64".toCharArray())), "HEX" }, null);
//$NON-NLS-1$
+ assertEquals("hello world", new
String(ObjectConverterUtil.convertToCharArray(result.getBinaryStream(), -1,
"ASCII")));
+ }
+
}
Modified:
trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java
===================================================================
---
trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java 2010-06-17
20:14:18 UTC (rev 2242)
+++
trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -209,6 +209,10 @@
assertEquals("a_u005F_x", XMLSystemFunctions.escapeName("a_x",
true));
}
+ @Test public void testNameEscaping2() throws Exception {
+ assertEquals("_u000A_", XMLSystemFunctions.escapeName(new String(new char[]
{10}), true));
+ }
+
@BeforeClass static public void setUpOnce() {
TimeZone.setDefault(TimeZone.getTimeZone("GMT-6:00"));
}
Modified: trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java 2010-06-17
20:14:18 UTC (rev 2242)
+++ trunk/engine/src/test/java/org/teiid/query/validator/TestValidator.java 2010-06-18
01:43:01 UTC (rev 2243)
@@ -2036,7 +2036,7 @@
}
@Test public void testDecode() throws Exception {
- helpValidate("select decode(e1, '?') from pm1.g1", new String[]
{"decode(e1, '?')"}, FakeMetadataFactory.example1Cached());
+ helpValidate("select to_bytes(e1, '?') from pm1.g1", new String[]
{"to_bytes(e1, '?')"}, FakeMetadataFactory.example1Cached());
}
}