Author: shawkins Date: 2012-07-05 16:35:48 -0400 (Thu, 05 Jul 2012) New Revision: 4223 Added: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/AbstractXMLStreamReader.java trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XMLEventStreamReader.java Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java trunk/engine/src/main/java/org/teiid/query/xquery/saxon/PathMapFilter.java trunk/engine/src/main/java/org/teiid/query/xquery/saxon/SaxonXQueryExpression.java trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XQueryEvaluator.java trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java Log: TEIID-2092 allowing for streaming json processing Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-07-05 20:35:48 UTC (rev 4223) @@ -28,6 +28,7 @@ <UL> <li><B>VDB Reuse</B> a vdb.xml can now declare imported vdbs to reuse metadata. <LI><B>Comparable Object</B> - the system property org.teiid.comparableObject can be set to use OBJECT values in comparison/sorting/grouping operations. It is expected that the object values correctly implement Comparable.compareTo. + <LI><B>Admin Metadata</B> - you can now retrieve metadata in DDL from the admin api via the getSchema method. </UL> <h2><a name="Compatibility">Compatibility Issues</a></h2> @@ -48,6 +49,7 @@ <li>VDB.Status now has three states - LOADING, ACTIVE, REMOVED. To check for validity use the isValid method, rather than checking for the VALID state. <li>The standalone and cli configuration files specify a setting for the teiid subsystem policy-decider-module. If a module is not specified, then data roles will not be checked. <li>local connections specifying a VDB version will wait for their VDB to finish loading before allowing a connection. + <li>jsonToXml document elements will contain xsi:type attribute values of decimal and boolean respectively for number and boolean json values to allow for differentiation from string values. <ul> <h4>from 7.x</h4> Modified: trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/main/java/org/teiid/query/eval/Evaluator.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -77,6 +77,7 @@ import org.teiid.query.xquery.saxon.XQueryEvaluator; import org.teiid.query.xquery.saxon.SaxonXQueryExpression.Result; import org.teiid.query.xquery.saxon.SaxonXQueryExpression.RowProcessor; +import org.teiid.translator.SourceSystemFunctions; import org.teiid.translator.WSConnection.Util; public class Evaluator { @@ -872,7 +873,7 @@ TeiidComponentException { Object contextItem = null; for (DerivedColumn passing : cols) { - Object value = this.evaluate(passing.getExpression(), tuple); + Object value = evaluateParameter(tuple, passing); if (passing.getAlias() == null) { contextItem = value; } else { @@ -882,6 +883,39 @@ return contextItem; } + private Object evaluateParameter(List<?> tuple, DerivedColumn passing) + throws ExpressionEvaluationException, BlockedException, + TeiidComponentException { + if (passing.getExpression() instanceof Function) { + Function f = (Function)passing.getExpression(); + //narrow optimization of json based documents to allow for lower overhead streaming + if (f.getFunctionDescriptor().getName().equalsIgnoreCase(SourceSystemFunctions.JSONTOXML)) { + String rootName = (String)this.evaluate(f.getArg(0), tuple); + Object lob = this.evaluate(f.getArg(1), tuple); + if (rootName == null || lob == null) { + return null; + } + try { + if (lob instanceof Blob) { + return XMLSystemFunctions.jsonToXml(context, rootName, (Blob)lob, true); + } + return XMLSystemFunctions.jsonToXml(context, rootName, (Clob)lob, true); + } catch (IOException e) { + throw new FunctionExecutionException(QueryPlugin.Event.TEIID30384, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30384, f.getFunctionDescriptor().getName())); + } catch (SQLException e) { + throw new FunctionExecutionException(QueryPlugin.Event.TEIID30384, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30384, f.getFunctionDescriptor().getName())); + } catch (TeiidProcessingException e) { + throw new FunctionExecutionException(QueryPlugin.Event.TEIID30384, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30384, f.getFunctionDescriptor().getName())); + } + } + } else if (passing.getExpression() instanceof XMLParse) { + XMLParse xmlParse = (XMLParse)passing.getExpression(); + xmlParse.setWellFormed(true); + } + Object value = this.evaluate(passing.getExpression(), tuple); + return value; + } + private Evaluator.NameValuePair<Object>[] getNameValuePairs(List<?> tuple, List<DerivedColumn> args, boolean xmlNames) throws ExpressionEvaluationException, BlockedException, TeiidComponentException { Evaluator.NameValuePair<Object>[] nameValuePairs = new Evaluator.NameValuePair[args.size()]; Modified: trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/main/java/org/teiid/query/function/source/SystemSource.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -974,6 +974,12 @@ new FunctionParameter("document", DataTypeManager.DefaultDataTypes.CLOB, QueryPlugin.Util.getString("SystemSource.xpath_param1")), //$NON-NLS-1$ //$NON-NLS-2$ new FunctionParameter("xpath", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.xpath_param2"))}, //$NON-NLS-1$ //$NON-NLS-2$ new FunctionParameter("result", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.xpathvalue_result")) ) ); //$NON-NLS-1$ //$NON-NLS-2$ + + functions.add(new FunctionMethod(SourceSystemFunctions.XPATHVALUE, QueryPlugin.Util.getString("SystemSource.xpathvalue_description"), XML, XML_FUNCTION_CLASS, "xpathValue", //$NON-NLS-1$ //$NON-NLS-2$ + new FunctionParameter[] { + new FunctionParameter("document", DataTypeManager.DefaultDataTypes.BLOB, QueryPlugin.Util.getString("SystemSource.xpath_param1")), //$NON-NLS-1$ //$NON-NLS-2$ + new FunctionParameter("xpath", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.xpath_param2"))}, //$NON-NLS-1$ //$NON-NLS-2$ + new FunctionParameter("result", DataTypeManager.DefaultDataTypes.STRING, QueryPlugin.Util.getString("SystemSource.xpathvalue_result")) ) ); //$NON-NLS-1$ //$NON-NLS-2$ functions.add(new FunctionMethod(SourceSystemFunctions.XPATHVALUE, QueryPlugin.Util.getString("SystemSource.xpathvalue_description"), XML, XML_FUNCTION_CLASS, "xpathValue", //$NON-NLS-1$ //$NON-NLS-2$ new FunctionParameter[] { Modified: trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/main/java/org/teiid/query/function/source/XMLSystemFunctions.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -47,18 +47,19 @@ import javax.xml.namespace.QName; import javax.xml.stream.EventFilter; import javax.xml.stream.FactoryConfigurationError; +import javax.xml.stream.Location; import javax.xml.stream.XMLEventFactory; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLEventWriter; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamWriter; import javax.xml.stream.events.XMLEvent; import javax.xml.transform.Source; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stax.StAXSource; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import javax.xml.xpath.XPathExpressionException; @@ -83,6 +84,7 @@ import org.teiid.common.buffer.FileStoreInputStreamFactory; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; +import org.teiid.core.TeiidRuntimeException; import org.teiid.core.types.ClobImpl; import org.teiid.core.types.ClobType; import org.teiid.core.types.SQLXMLImpl; @@ -90,6 +92,7 @@ import org.teiid.core.types.XMLTranslator; import org.teiid.core.types.XMLType; import org.teiid.core.types.XMLType.Type; +import org.teiid.jdbc.TeiidSQLException; import org.teiid.query.QueryPlugin; import org.teiid.query.eval.Evaluator; import org.teiid.query.function.CharsetUtils; @@ -110,50 +113,54 @@ private static final Charset UTF_16LE = Charset.forName("UTF-16LE"); //$NON-NLS-1$ private static final Charset UTF_8 = Charset.forName("UTF-8"); //$NON-NLS-1$ - //TODO: this could be done fully streaming without holding the intermediate xml output private static final class JsonToXmlContentHandler implements - ContentHandler { - private final XMLStreamWriter streamWriter; + ContentHandler, XMLEventReader { + private Reader reader; + private JSONParser parser; + private XMLEventFactory eventFactory; + + private LinkedList<String> nameStack = new LinkedList<String>(); + private LinkedList<XMLEvent> eventStack = new LinkedList<XMLEvent>(); + private boolean rootArray; - private LinkedList<String> nameStack = new LinkedList<String>(); + private boolean end; + private boolean declaredNs; private JsonToXmlContentHandler(String rootName, - XMLStreamWriter streamWriter) { - this.streamWriter = streamWriter; - this.nameStack.push(rootName); + Reader reader, JSONParser parser, XMLEventFactory eventFactory) { + this.nameStack.push(escapeName(rootName, true)); + this.reader = reader; + this.eventFactory = eventFactory; + this.parser = parser; } @Override public boolean startObjectEntry(String key) throws org.json.simple.parser.ParseException, IOException { - this.nameStack.push(key); - return true; + this.nameStack.push(escapeName(key, true)); + return false; } @Override public boolean startObject() throws org.json.simple.parser.ParseException, IOException { start(); - return true; + return false; } - private void start() - throws IOException { - try { - streamWriter.writeStartElement(escapeName(this.nameStack.peek(), true)); - } catch (XMLStreamException e) { - throw new IOException(e); + private void start() { + eventStack.add(eventFactory.createStartElement("", "", nameStack.peek())); //$NON-NLS-1$ //$NON-NLS-2$ + if (!declaredNs) { + eventStack.add(eventFactory.createNamespace("xsi", XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI)); //$NON-NLS-1$ + declaredNs = true; } } @Override public void startJSON() throws org.json.simple.parser.ParseException, IOException { - try { - streamWriter.writeStartDocument(); - } catch (XMLStreamException e) { - throw new IOException(e); - } + //specify the defaults, since different providers emit/omit differently + eventStack.add(eventFactory.createStartDocument("UTF-8", "1.0")); //$NON-NLS-1$ //$NON-NLS-2$ } @Override @@ -163,58 +170,55 @@ this.rootArray = true; start(); } - return true; + return false; } @Override public boolean primitive(Object value) throws org.json.simple.parser.ParseException, IOException { start(); - try { - if (value != null) { - streamWriter.writeCharacters(value.toString()); - } else { - streamWriter.writeNamespace("xsi", XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI); //$NON-NLS-1$ - streamWriter.writeAttribute("xsi", XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI, "nil", "true"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + if (value != null) { + String type = "decimal"; //$NON-NLS-1$ + if (value instanceof String) { + type = null; + } else if (value instanceof Boolean) { + type = "boolean"; //$NON-NLS-1$ } - } catch (XMLStreamException e) { - throw new IOException(e); + if (type != null) { + //we need to differentiate boolean/decimal entries from their string counter parts + eventStack.add(eventFactory.createAttribute("xsi", XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI, "type", type)); //$NON-NLS-1$ //$NON-NLS-2$ + } + eventStack.add(eventFactory.createCharacters(value.toString())); + } else { + eventStack.add(eventFactory.createAttribute("xsi", XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI, "nil", "true")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } end(); - return true; + return true; //return true, otherwise we don't get the endObjectEntry } - private void end() - throws IOException { - try { - streamWriter.writeEndElement(); - } catch (XMLStreamException e) { - throw new IOException(e); - } + private void end() { + eventStack.add(eventFactory.createEndElement("", "", nameStack.peek())); //$NON-NLS-1$ //$NON-NLS-2$ } @Override public boolean endObjectEntry() throws org.json.simple.parser.ParseException, IOException { this.nameStack.pop(); - return true; + return false; } @Override public boolean endObject() throws org.json.simple.parser.ParseException, IOException { end(); - return true; + return false; } @Override public void endJSON() throws org.json.simple.parser.ParseException, IOException { - try { - streamWriter.writeEndDocument(); - } catch (XMLStreamException e) { - throw new IOException(e); - } + this.eventStack.add(eventFactory.createEndDocument()); + end = true; } @Override @@ -223,8 +227,76 @@ if (this.nameStack.size() == 1 && rootArray) { end(); } - return true; + return false; } + + @Override + public void close() throws XMLStreamException { + try { + //this is explicitly against the javadoc, but + //it's our only chance to close the reader + this.reader.close(); + } catch (IOException e) { + } + } + + @Override + public String getElementText() throws XMLStreamException { + throw new UnsupportedOperationException(); + } + + @Override + public Object getProperty(String name) throws IllegalArgumentException { + return null; + } + + @Override + public boolean hasNext() { + return !eventStack.isEmpty() || !end; + } + + @Override + public XMLEvent nextEvent() throws XMLStreamException { + while (eventStack.isEmpty() && !end) { + try { + parser.parse(reader, this, true); + } catch (IOException e) { + throw new XMLStreamException(e); + } catch (ParseException e) { + throw new XMLStreamException(e); + } + } + return eventStack.remove(); + } + + @Override + public XMLEvent nextTag() throws XMLStreamException { + throw new UnsupportedOperationException(); + } + + @Override + public XMLEvent peek() throws XMLStreamException { + if (hasNext()) { + XMLEvent next = next(); + this.eventStack.push(next); + return next; + } + return null; + } + + @Override + public XMLEvent next() { + try { + return nextEvent(); + } catch (XMLStreamException e) { + throw new TeiidRuntimeException(e); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } } private static ThreadLocal<TransformerFactory> threadLocalTransformerFactory = new ThreadLocal<TransformerFactory>() { @@ -239,8 +311,13 @@ }; static ThreadLocal<XMLEventFactory> threadLocalEventtFactory = new ThreadLocal<XMLEventFactory>() { protected XMLEventFactory initialValue() { - return XMLEventFactory.newInstance(); + return XMLEventFactory.newFactory(); } + public XMLEventFactory get() { + XMLEventFactory eventFactory = super.get(); + eventFactory.setLocation(null); + return eventFactory; + } }; private static final String P_OUTPUT_VALIDATE_STRUCTURE = "com.ctc.wstx.outputValidateStructure"; //$NON-NLS-1$ static XMLOutputFactory newXmlOutputFactory() throws FactoryConfigurationError { @@ -436,7 +513,7 @@ fs.remove(); throw new TeiidProcessingException(QueryPlugin.Event.TEIID30437, e); } - eventFactory = XMLEventFactory.newInstance(); + eventFactory = threadLocalEventtFactory.get(); } public void addValue(Object object) throws TeiidProcessingException { @@ -712,6 +789,10 @@ } public static SQLXML jsonToXml(CommandContext context, final String rootName, final Blob json) throws TeiidComponentException, TeiidProcessingException, SQLException, IOException { + return jsonToXml(context, rootName, json, false); + } + + public static SQLXML jsonToXml(CommandContext context, final String rootName, final Blob json, boolean stream) throws TeiidComponentException, TeiidProcessingException, SQLException, IOException { InputStream is = json.getBinaryStream(); PushbackInputStream pStream = new PushbackInputStream(is, 4); byte[] encoding = new byte[3]; @@ -734,33 +815,81 @@ } } Reader r = new InputStreamReader(pStream, charset); - return jsonToXml(context, rootName, r); + return jsonToXml(context, rootName, r, stream); } - + public static SQLXML jsonToXml(CommandContext context, final String rootName, final Clob json) throws TeiidComponentException, TeiidProcessingException, SQLException { - return jsonToXml(context, rootName, json.getCharacterStream()); + return jsonToXml(context, rootName, json, false); } - + + public static SQLXML jsonToXml(CommandContext context, final String rootName, final Clob json, boolean stream) throws TeiidComponentException, TeiidProcessingException, SQLException { + return jsonToXml(context, rootName, json.getCharacterStream(), stream); + } + private static SQLXML jsonToXml(CommandContext context, - final String rootName, final Reader r) throws TeiidComponentException, + final String rootName, final Reader r, boolean stream) throws TeiidComponentException, TeiidProcessingException { + JSONParser parser = new JSONParser(); + final JsonToXmlContentHandler reader = new JsonToXmlContentHandler(rootName, r, parser, threadLocalEventtFactory.get()); + + if (stream) { + //jre 1.7 event logic does not set a dummy location and throws an NPE in StAXSource, so we explicitly set a location + reader.eventFactory.setLocation(new Location() { + + @Override + public String getSystemId() { + return null; + } + + @Override + public String getPublicId() { + return null; + } + + @Override + public int getLineNumber() { + return -1; + } + + @Override + public int getColumnNumber() { + return -1; + } + + @Override + public int getCharacterOffset() { + return -1; + } + }); + return new SQLXMLImpl() { + @SuppressWarnings("unchecked") + public <T extends Source> T getSource(Class<T> sourceClass) throws SQLException { + if (sourceClass == null || sourceClass == StAXSource.class) { + StAXSource source; + try { + source = new StAXSource(reader); + } catch (XMLStreamException e) { + throw TeiidSQLException.create(e); + } + return (T) source; + } + throw new AssertionError("unsupported source type"); //$NON-NLS-1$ + } + }; + } XMLType result = new XMLType(XMLSystemFunctions.saveToBufferManager(context.getBufferManager(), new XMLTranslator() { @Override public void translate(Writer writer) throws TransformerException, IOException { try { - JSONParser parser = new JSONParser(); XMLOutputFactory factory = getOutputFactory(); - final XMLStreamWriter streamWriter = factory.createXMLStreamWriter(writer); - - parser.parse(r, new JsonToXmlContentHandler(escapeName(rootName, true), streamWriter)); - + final XMLEventWriter streamWriter = factory.createXMLEventWriter(writer); + + streamWriter.add(reader); streamWriter.flush(); //woodstox needs a flush rather than a close } catch (XMLStreamException e) { throw new TransformerException(e); - } catch (ParseException e) { - throw new TransformerException(e); } finally { try { r.close(); Added: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/AbstractXMLStreamReader.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/xquery/saxon/AbstractXMLStreamReader.java (rev 0) +++ trunk/engine/src/main/java/org/teiid/query/xquery/saxon/AbstractXMLStreamReader.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -0,0 +1,180 @@ +/* + * Copyright 2002-2009 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.teiid.query.xquery.saxon; + +import javax.xml.namespace.QName; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * Abstract base class for <code>XMLStreamReader</code>s. + * + * @author Arjen Poutsma + * @since 3.0 + */ +abstract class AbstractXMLStreamReader implements XMLStreamReader { + + public String getElementText() throws XMLStreamException { + if (getEventType() != XMLStreamConstants.START_ELEMENT) { + throw new XMLStreamException("parser must be on START_ELEMENT to read next text", getLocation()); + } + int eventType = next(); + StringBuilder builder = new StringBuilder(); + while (eventType != XMLStreamConstants.END_ELEMENT) { + if (eventType == XMLStreamConstants.CHARACTERS || eventType == XMLStreamConstants.CDATA || + eventType == XMLStreamConstants.SPACE || eventType == XMLStreamConstants.ENTITY_REFERENCE) { + builder.append(getText()); + } + else if (eventType == XMLStreamConstants.PROCESSING_INSTRUCTION || + eventType == XMLStreamConstants.COMMENT) { + // skipping + } + else if (eventType == XMLStreamConstants.END_DOCUMENT) { + throw new XMLStreamException("unexpected end of document when reading element text content", + getLocation()); + } + else if (eventType == XMLStreamConstants.START_ELEMENT) { + throw new XMLStreamException("element text content may not contain START_ELEMENT", getLocation()); + } + else { + throw new XMLStreamException("Unexpected event type " + eventType, getLocation()); + } + eventType = next(); + } + return builder.toString(); + } + + public String getAttributeLocalName(int index) { + return getAttributeName(index).getLocalPart(); + } + + public String getAttributeNamespace(int index) { + return getAttributeName(index).getNamespaceURI(); + } + + public String getAttributePrefix(int index) { + return getAttributeName(index).getPrefix(); + } + + public String getNamespaceURI() { + int eventType = getEventType(); + if (eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT) { + return getName().getNamespaceURI(); + } + else { + throw new IllegalStateException("parser must be on START_ELEMENT or END_ELEMENT state"); + } + } + + public String getNamespaceURI(String prefix) { + return getNamespaceContext().getNamespaceURI(prefix); + } + + public boolean hasText() { + int eventType = getEventType(); + return eventType == XMLStreamConstants.SPACE || eventType == XMLStreamConstants.CHARACTERS || + eventType == XMLStreamConstants.COMMENT || eventType == XMLStreamConstants.CDATA || + eventType == XMLStreamConstants.ENTITY_REFERENCE; + } + + public String getPrefix() { + int eventType = getEventType(); + if (eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT) { + return getName().getPrefix(); + } + else { + throw new IllegalStateException("parser must be on START_ELEMENT or END_ELEMENT state"); + } + } + + public boolean hasName() { + int eventType = getEventType(); + return eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT; + } + + public boolean isWhiteSpace() { + return getEventType() == XMLStreamConstants.SPACE; + } + + public boolean isStartElement() { + return getEventType() == XMLStreamConstants.START_ELEMENT; + } + + public boolean isEndElement() { + return getEventType() == XMLStreamConstants.END_ELEMENT; + } + + public boolean isCharacters() { + return getEventType() == XMLStreamConstants.CHARACTERS; + } + + public int nextTag() throws XMLStreamException { + int eventType = next(); + while (eventType == XMLStreamConstants.CHARACTERS && isWhiteSpace() || + eventType == XMLStreamConstants.CDATA && isWhiteSpace() || eventType == XMLStreamConstants.SPACE || + eventType == XMLStreamConstants.PROCESSING_INSTRUCTION || eventType == XMLStreamConstants.COMMENT) { + eventType = next(); + } + if (eventType != XMLStreamConstants.START_ELEMENT && eventType != XMLStreamConstants.END_ELEMENT) { + throw new XMLStreamException("expected start or end tag", getLocation()); + } + return eventType; + } + + public void require(int expectedType, String namespaceURI, String localName) throws XMLStreamException { + int eventType = getEventType(); + if (eventType != expectedType) { + throw new XMLStreamException("Expected [" + expectedType + "] but read [" + eventType + "]"); + } + } + + public String getAttributeValue(String namespaceURI, String localName) { + for (int i = 0; i < getAttributeCount(); i++) { + QName name = getAttributeName(i); + if (name.getLocalPart().equals(localName) && + (namespaceURI == null || name.getNamespaceURI().equals(namespaceURI))) { + return getAttributeValue(i); + } + } + return null; + } + + public boolean hasNext() throws XMLStreamException { + return getEventType() != END_DOCUMENT; + } + + public String getLocalName() { + return getName().getLocalPart(); + } + + public char[] getTextCharacters() { + return getText().toCharArray(); + } + + public int getTextCharacters(int sourceStart, char[] target, int targetStart, int length) + throws XMLStreamException { + char[] source = getTextCharacters(); + length = Math.min(length, source.length); + System.arraycopy(source, sourceStart, target, targetStart, length); + return length; + } + + public int getTextLength() { + return getText().length(); + } +} Property changes on: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/AbstractXMLStreamReader.java ___________________________________________________________________ Added: svn:mime-type + text/plain Modified: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/PathMapFilter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/xquery/saxon/PathMapFilter.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/main/java/org/teiid/query/xquery/saxon/PathMapFilter.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -86,7 +86,7 @@ } } - + private boolean closed; private LinkedList<MatchContext> matchContext = new LinkedList<MatchContext>(); public PathMapFilter(PathMapRoot root) { @@ -192,5 +192,13 @@ super.startContent(); } } + + @Override + public void close() throws XPathException { + if (!closed) { + super.close(); + closed = true; + } + } } \ No newline at end of file Modified: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/SaxonXQueryExpression.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/xquery/saxon/SaxonXQueryExpression.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/main/java/org/teiid/query/xquery/saxon/SaxonXQueryExpression.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -33,10 +33,12 @@ import java.util.Map; import java.util.Properties; +import javax.xml.stream.XMLStreamException; import javax.xml.transform.ErrorListener; import javax.xml.transform.OutputKeys; import javax.xml.transform.Source; import javax.xml.transform.TransformerException; +import javax.xml.transform.stax.StAXSource; import net.sf.saxon.Configuration; import net.sf.saxon.expr.AxisExpression; @@ -115,6 +117,20 @@ public void close() { for (Source source : sources) { Util.closeSource(source); + if (source instanceof StAXSource) { + StAXSource ss = (StAXSource)source; + if (ss.getXMLEventReader() != null) { + try { + ss.getXMLEventReader().close(); + } catch (XMLStreamException e) { + } + } else { + try { + ss.getXMLStreamReader().close(); + } catch (XMLStreamException e) { + } + } + } } if (iter != null) { iter.close(); Added: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XMLEventStreamReader.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XMLEventStreamReader.java (rev 0) +++ trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XMLEventStreamReader.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -0,0 +1,256 @@ +/* + * Copyright 2002-2010 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.teiid.query.xquery.saxon; + +import java.util.Iterator; + +import javax.xml.namespace.NamespaceContext; +import javax.xml.namespace.QName; +import javax.xml.stream.Location; +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.Attribute; +import javax.xml.stream.events.Namespace; +import javax.xml.stream.events.ProcessingInstruction; +import javax.xml.stream.events.StartDocument; +import javax.xml.stream.events.XMLEvent; + +/** + * Implementation of the {@link javax.xml.stream.XMLStreamReader} interface that wraps a {@link XMLEventReader}. Useful, + * because the StAX {@link javax.xml.stream.XMLInputFactory} allows one to create a event reader from a stream reader, + * but not vice-versa. + * + * @author Arjen Poutsma + * @since 3.0 + * @see StaxUtils#createEventStreamReader(javax.xml.stream.XMLEventReader) + */ +class XMLEventStreamReader extends AbstractXMLStreamReader { + + private XMLEvent event; + + private final XMLEventReader eventReader; + + XMLEventStreamReader(XMLEventReader eventReader) throws XMLStreamException { + this.eventReader = eventReader; + event = eventReader.nextEvent(); + } + + public boolean isStandalone() { + if (event.isStartDocument()) { + return ((StartDocument) event).isStandalone(); + } + else { + throw new IllegalStateException(); + } + } + + public String getVersion() { + if (event.isStartDocument()) { + return ((StartDocument) event).getVersion(); + } + else { + return null; + } + } + + public int getTextStart() { + return 0; + } + + public String getText() { + if (event.isCharacters()) { + return event.asCharacters().getData(); + } + else { + throw new IllegalStateException(); + } + } + + public String getPITarget() { + if (event.isProcessingInstruction()) { + return ((ProcessingInstruction) event).getTarget(); + } + else { + throw new IllegalStateException(); + } + } + + public String getPIData() { + if (event.isProcessingInstruction()) { + return ((ProcessingInstruction) event).getData(); + } + else { + throw new IllegalStateException(); + } + } + + public int getNamespaceCount() { + Iterator namespaces; + if (event.isStartElement()) { + namespaces = event.asStartElement().getNamespaces(); + } + else if (event.isEndElement()) { + namespaces = event.asEndElement().getNamespaces(); + } + else { + throw new IllegalStateException(); + } + return countIterator(namespaces); + } + + public NamespaceContext getNamespaceContext() { + if (event.isStartElement()) { + return event.asStartElement().getNamespaceContext(); + } + else { + throw new IllegalStateException(); + } + } + + public QName getName() { + if (event.isStartElement()) { + return event.asStartElement().getName(); + } + else if (event.isEndElement()) { + return event.asEndElement().getName(); + } + else { + throw new IllegalStateException(); + } + } + + public Location getLocation() { + return event.getLocation(); + } + + public int getEventType() { + return event.getEventType(); + } + + public String getEncoding() { + return null; + } + + public String getCharacterEncodingScheme() { + return null; + } + + public int getAttributeCount() { + if (!event.isStartElement()) { + throw new IllegalStateException(); + } + Iterator attributes = event.asStartElement().getAttributes(); + return countIterator(attributes); + } + + public void close() throws XMLStreamException { + eventReader.close(); + } + + public QName getAttributeName(int index) { + return getAttribute(index).getName(); + } + + public String getAttributeType(int index) { + return getAttribute(index).getDTDType(); + } + + public String getAttributeValue(int index) { + return getAttribute(index).getValue(); + } + + public String getNamespacePrefix(int index) { + return getNamespace(index).getPrefix(); + } + + public String getNamespaceURI(int index) { + return getNamespace(index).getNamespaceURI(); + } + + public Object getProperty(String name) throws IllegalArgumentException { + return eventReader.getProperty(name); + } + + public boolean isAttributeSpecified(int index) { + return getAttribute(index).isSpecified(); + } + + public int next() throws XMLStreamException { + event = eventReader.nextEvent(); + return event.getEventType(); + } + + public boolean standaloneSet() { + if (event.isStartDocument()) { + return ((StartDocument) event).standaloneSet(); + } + else { + throw new IllegalStateException(); + } + } + + private int countIterator(Iterator iterator) { + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + return count; + } + + private Attribute getAttribute(int index) { + if (!event.isStartElement()) { + throw new IllegalStateException(); + } + int count = 0; + Iterator attributes = event.asStartElement().getAttributes(); + while (attributes.hasNext()) { + Attribute attribute = (Attribute) attributes.next(); + if (count == index) { + return attribute; + } + else { + count++; + } + } + throw new IllegalArgumentException(); + } + + private Namespace getNamespace(int index) { + Iterator namespaces; + if (event.isStartElement()) { + namespaces = event.asStartElement().getNamespaces(); + } + else if (event.isEndElement()) { + namespaces = event.asEndElement().getNamespaces(); + } + else { + throw new IllegalStateException(); + } + int count = 0; + while (namespaces.hasNext()) { + Namespace namespace = (Namespace) namespaces.next(); + if (count == index) { + return namespace; + } + else { + count++; + } + } + throw new IllegalArgumentException(); + } +} Property changes on: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XMLEventStreamReader.java ___________________________________________________________________ Added: svn:mime-type + text/plain Modified: trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XQueryEvaluator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XQueryEvaluator.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/main/java/org/teiid/query/xquery/saxon/XQueryEvaluator.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -27,12 +27,16 @@ import java.sql.SQLXML; import java.util.Map; +import javax.xml.stream.XMLStreamException; import javax.xml.transform.Source; import javax.xml.transform.TransformerException; +import javax.xml.transform.stax.StAXSource; import net.sf.saxon.AugmentedSource; import net.sf.saxon.Configuration; import net.sf.saxon.event.ProxyReceiver; +import net.sf.saxon.evpull.PullEventSource; +import net.sf.saxon.evpull.StaxToEventBridge; import net.sf.saxon.om.DocumentInfo; import net.sf.saxon.om.NodeInfo; import net.sf.saxon.query.DynamicQueryContext; @@ -83,6 +87,7 @@ if(value instanceof SQLXML) { value = XMLSystemFunctions.convertToSource(value); result.sources.add((Source)value); + value = wrapStax((Source)value, xquery.getConfig()); } else if (value instanceof java.util.Date) { value = XMLSystemFunctions.convertToAtomicValue(value); } @@ -94,6 +99,7 @@ if (context != null) { Source source = XMLSystemFunctions.convertToSource(context); result.sources.add(source); + source = wrapStax(source, xquery.getConfig()); if (xquery.contextRoot != null) { //create our own filter as this logic is not provided in the free saxon ProxyReceiver filter = new PathMapFilter(xquery.contextRoot); @@ -158,6 +164,27 @@ } } + private static Source wrapStax(Source value, Configuration config) throws TeiidProcessingException { + if (value instanceof StAXSource) { + //saxon doesn't like staxsources + StaxToEventBridge sb = new StaxToEventBridge(); + sb.setPipelineConfiguration(config.makePipelineConfiguration()); + StAXSource staxSource = (StAXSource)value; + if (staxSource.getXMLEventReader() != null) { + try { + sb.setXMLStreamReader(new XMLEventStreamReader(staxSource.getXMLEventReader())); + } catch (XMLStreamException e) { + //should not happen as the StAXSource already peeked + throw new TeiidProcessingException(e); + } + } else { + sb.setXMLStreamReader(staxSource.getXMLStreamReader()); + } + value = new PullEventSource(sb); + } + return value; + } + /** * Converts a xom node into something readable by Saxon * @param node Modified: trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/test/java/org/teiid/query/function/source/TestXMLSystemFunctions.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -194,7 +194,7 @@ @Test public void testJsonToXml() throws Exception { String json = "[0,{\"1\":{\"2\":{\"3\":{\"4\":[5,{\"6\":7}]}}}}]"; - String expected = "<?xml version=\"1.0\" ?><Array><Array>0</Array><Array><_u0031_><_u0032_><_u0033_><_u0034_>5</_u0034_><_u0034_><_u0036_>7</_u0036_></_u0034_></_u0033_></_u0032_></_u0031_></Array></Array>"; + String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Array xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><... xsi:type=\"decimal\">0</Array><Array><_u0031_><_u0032_><_u0033_><_u0034_ xsi:type=\"decimal\">5</_u0034_><_u0034_><_u0036_ xsi:type=\"decimal\">7</_u0036_></_u0034_></_u0033_></_u0032_></_u0031_></Array></Array>"; helpTestJson(json, "Array", expected); } @@ -214,25 +214,25 @@ @Test public void testJsonToXml1() throws Exception { String json = "{ \"firstName\": \"John\", \"lastName\": \"Smith\", \"age\": 25, \"address\": { \"streetAddress\": \"21 2nd Street\", \"city\": \"New York\", \"state\": \"NY\", "+ "\"postalCode\": \"10021\" }, \"phoneNumber\": [ { \"type\": \"home\", \"number\": \"212 555-1234\" }, { \"type\": \"fax\", \"number\": \"646 555-4567\" } ] }"; - String expected = "<?xml version=\"1.0\" ?><Person><firstName>John</firstName><lastName>Smith</lastName><age>25</age><address><streetAddress>21 2nd Street</streetAddress><city>New York</city><state>NY</state><postalCode>10021</postalCode></address><phoneNumber><type>home</type><number>212 555-1234</number></phoneNumber><phoneNumber><type>fax</type><number>646 555-4567</number></phoneNumber></Person>"; + String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Person xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><... xsi:type=\"decimal\">25</age><address><streetAddress>21 2nd Street</streetAddress><city>New York</city><state>NY</state><postalCode>10021</postalCode></address><phoneNumber><type>home</type><number>212 555-1234</number></phoneNumber><phoneNumber><type>fax</type><number>646 555-4567</number></phoneNumber></Person>"; helpTestJson(json, "Person", expected); } @Test public void testJsonToXml2() throws Exception { String json = "{ \"firstName\": null }"; - String expected = "<?xml version=\"1.0\" ?><Person><firstName xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:nil=\"true\"></firstName></Person>"; + String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Person xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><... xsi:nil=\"true\"></firstName></Person>"; helpTestJson(json, "Person", expected); } @Test public void testJsonToXml3() throws Exception { String json = "{ \"kids\":[{ \"firstName\" : \"George\" }, { \"firstName\" : \"Jerry\" }]}"; - String expected = "<?xml version=\"1.0\" ?><Person><kids><firstName>George</firstName></kids><kids><firstName>Jerry</firstName></kids></Person>"; + String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Person xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><...; helpTestJson(json, "Person", expected); } @Test public void testJsonToXml4() throws Exception { String json = "{ \"kids\":[{ \"firstName\" : \"George\" }, { \"firstName\" : \"Jerry\" }]}"; - String expected = "<?xml version=\"1.0\" ?><Person><kids><firstName>George</firstName></kids><kids><firstName>Jerry</firstName></kids></Person>"; + String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Person xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><...; helpTestJson(json, "Person", expected); } @@ -242,7 +242,7 @@ */ @Test public void testJsonToXml5() throws Exception { String json = "[[],{\"x\": 1},[]]"; - String expected = "<?xml version=\"1.0\" ?><Person><Person></Person><Person><x>1</x></Person><Person></Person></Person>"; + String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Person xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><... xsi:type=\"decimal\">1</x></Person><Person></Person></Person>"; helpTestJson(json, "Person", expected); } Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java 2012-07-05 15:43:21 UTC (rev 4222) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java 2012-07-05 20:35:48 UTC (rev 4223) @@ -27,6 +27,7 @@ import java.io.FileInputStream; import java.io.InputStreamReader; import java.nio.charset.Charset; +import java.sql.Blob; import java.sql.Timestamp; import java.util.Arrays; import java.util.List; @@ -515,4 +516,18 @@ process(sql, expected); } + @Test public void testJsonStreamingXmlTable() throws Exception { + String sql = "select * from xmltable('/Person/phoneNumber' passing jsontoxml('Person', cast(? as blob)) columns x string path 'type', y string path 'number') as x"; //$NON-NLS-1$ + + List<?>[] expected = new List<?>[] { + Arrays.asList("home", "212 555-1234"), + Arrays.asList("fax", "646 555-4567"), + }; + + Blob b = BlobType.createBlob(("{ \"firstName\": \"John\", \"lastName\": \"Smith\", \"age\": 25, \"address\": { \"streetAddress\": \"21 2nd Street\", \"city\": \"New York\", \"state\": \"NY\", "+ + "\"postalCode\": \"10021\" }, \"phoneNumber\": [ { \"type\": \"home\", \"number\": \"212 555-1234\" }, { \"type\": \"fax\", \"number\": \"646 555-4567\" } ] }").getBytes(Charset.forName("UTF-8"))); + + processPreparedStatement(sql, expected, dataManager, new DefaultCapabilitiesFinder(), RealMetadataFactory.example1Cached(), Arrays.asList(b)); + } + }