[teiid-commits] teiid SVN: r1723 - in trunk: client/src/test/java/com/metamatrix/common/lob and 34 other directories.

Author: shawkins Date: 2010-01-08 14:51:25 -0500 (Fri, 08 Jan 2010) New Revision: 1723 Added: trunk/common-core/src/main/java/com/metamatrix/common/types/InputStreamFactory.java trunk/common-core/src/main/java/com/metamatrix/common/types/StandardXMLTranslator.java trunk/engine/src/main/java/com/metamatrix/common/buffer/FileStore.java Removed: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java trunk/common-core/src/main/java/com/metamatrix/common/types/BaseXMLTranslator.java trunk/common-core/src/main/java/com/metamatrix/common/types/XMLDomSourceTranslator.java trunk/common-core/src/main/java/com/metamatrix/common/types/XMLReaderFactory.java trunk/common-core/src/main/java/com/metamatrix/common/types/XMLSAXSourceTranslator.java trunk/common-core/src/main/java/com/metamatrix/common/types/XMLStreamSourceTranslator.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/CachingConnector.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/cache/ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceNotFoundException.java trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/ trunk/engine/src/main/java/com/metamatrix/connector/ trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXMLTranslator.java trunk/engine/src/test/java/com/metamatrix/common/buffer/TestTupleSourceID.java trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/ Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMResultSet.java trunk/client/src/main/java/com/metamatrix/common/lob/LobChunkInputStream.java trunk/client/src/test/java/com/metamatrix/common/lob/TestLobChunkInputStream.java trunk/common-core/src/main/java/com/metamatrix/common/types/DataTypeManager.java trunk/common-core/src/main/java/com/metamatrix/common/types/SQLXMLImpl.java trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java trunk/common-core/src/main/java/com/metamatrix/common/types/XMLTranslator.java trunk/common-core/src/main/java/com/metamatrix/common/types/basic/StringToSQLXMLTransform.java trunk/common-core/src/test/java/com/metamatrix/common/types/TestSQLXMLImpl.java trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLStreamSourceTranslator.java trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnection.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnectorState.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/AbstractCachingConnector.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectionImpl.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectorStateImpl.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectionImpl.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectorState.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileExecution.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectionImpl.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectorState.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPRequest.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectionImpl.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectorState.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPRequest.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/streaming/BaseStreamingExecution.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/XMLSourceExecution.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/file/FileExecution.java trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/soap/SoapExecution.java trunk/connectors/connector-xml/src/test/java/com/metamatrix/connector/xml/base/TestXMLConnectorState.java trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java trunk/engine/src/main/java/com/metamatrix/query/function/FunctionTree.java trunk/engine/src/main/java/com/metamatrix/query/function/metadata/FunctionMethod.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureCriteriaProcessor.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java trunk/engine/src/main/java/com/metamatrix/query/rewriter/QueryRewriter.java trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java trunk/engine/src/main/java/com/metamatrix/query/xquery/XQueryExpression.java trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestAggregateProcessing.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestGroupingNode.java trunk/engine/src/test/java/com/metamatrix/query/xquery/TestXQueryEngine.java Log: TEIID-917 TEIID-913 refactoring to further simplify buffermanagement and xml stream handling Modified: trunk/client/src/main/java/com/metamatrix/common/lob/LobChunkInputStream.java =================================================================== --- trunk/client/src/main/java/com/metamatrix/common/lob/LobChunkInputStream.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/client/src/main/java/com/metamatrix/common/lob/LobChunkInputStream.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,13 +22,13 @@ package com.metamatrix.common.lob; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; +import com.metamatrix.common.types.Streamable; import com.metamatrix.core.CorePlugin; @@ -38,16 +38,13 @@ * stream the lob data. */ public class LobChunkInputStream extends InputStream { - LobChunkProducer reader; + private LobChunkProducer reader; - byte[] byteData = null; - int currentCounter = 0; - boolean lastChunk = false; - int availableCounter = 0; - boolean closed = false; + private byte[] byteData = null; + private int currentCounter = 0; + private boolean lastChunk = false; + private boolean closed = false; - byte[] contents = null; - public LobChunkInputStream(LobChunkProducer reader) { this.reader = reader; } @@ -56,37 +53,19 @@ if (this.closed) { throw new IllegalStateException(CorePlugin.Util.getString("stream_closed")); //$NON-NLS-1$ } - if (this.availableCounter == 0) { + while (this.byteData == null || this.byteData.length <= currentCounter) { if (this.lastChunk) { // we are done return -1; } - fetchNextChunk(); - } - - // so we have data - int ret = -1; - if (this.availableCounter > 0) { - ret = (byteData[currentCounter++] & 0xFF); - this.availableCounter--; - } - return ret; - } - - void fetchNextChunk() throws IOException { - LobChunk value = this.reader.getNextChunk(); - if (value != null) { + LobChunk value = this.reader.getNextChunk(); this.lastChunk = value.isLast(); this.byteData = value.getBytes(); this.currentCounter = 0; - this.availableCounter = this.byteData.length; - } else { - throw new IOException(CorePlugin.Util.getString("lob.invaliddata")); //$NON-NLS-1$ } - } - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); + // so we have data + return (byteData[currentCounter++] & 0xFF); } /** @@ -97,30 +76,10 @@ this.reader.close(); } - /** - * Get the byte contents of the input stream. use caution as this may use up VM memory as - * the contents are loaded into memory. - */ - public byte[] getByteContents() throws IOException { - if (this.contents == null) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(100*1024); - byte[] buf = new byte[100*1024]; - int read = read(buf); - while(read != -1) { - bos.write(buf, 0, read); - read = read(buf); - } - close(); - this.contents = bos.toByteArray(); - bos.close(); - } - return this.contents; - } - /** - * @return a valid UTF16 based reader + * @return a valid reader */ - public Reader getUTF16Reader() { - return new InputStreamReader(this, Charset.forName("UTF-16")); //$NON-NLS-1$ + public Reader getReader() { + return new InputStreamReader(this, Charset.forName(Streamable.ENCODING)); } } Modified: trunk/client/src/test/java/com/metamatrix/common/lob/TestLobChunkInputStream.java =================================================================== --- trunk/client/src/test/java/com/metamatrix/common/lob/TestLobChunkInputStream.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/client/src/test/java/com/metamatrix/common/lob/TestLobChunkInputStream.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -30,6 +30,7 @@ import junit.framework.TestCase; +import com.metamatrix.core.util.ObjectConverterUtil; import com.metamatrix.core.util.UnitTestUtil; public class TestLobChunkInputStream extends TestCase { @@ -53,7 +54,7 @@ LobChunkInputStream reader = new LobChunkInputStream(chunkProducer); - String fileContents = new String(reader.getByteContents()); + String fileContents = new String(ObjectConverterUtil.convertToByteArray(reader)); assertEquals(readFile(new FileInputStream(f)), fileContents); } Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java =================================================================== --- trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -104,7 +104,7 @@ * <code>CLOB</code> value */ public Reader getCharacterStream() throws SQLException { - return new LobChunkInputStream(lobChunckFactory.getLobChunkProducer()).getUTF16Reader(); + return new LobChunkInputStream(lobChunckFactory.getLobChunkProducer()).getReader(); } /** Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMResultSet.java =================================================================== --- trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMResultSet.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMResultSet.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,6 +22,7 @@ package com.metamatrix.jdbc; +import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; @@ -49,8 +50,11 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; +import com.metamatrix.common.lob.LobChunkInputStream; import com.metamatrix.common.types.BlobType; import com.metamatrix.common.types.ClobType; +import com.metamatrix.common.types.InputStreamFactory; +import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.Streamable; import com.metamatrix.common.types.XMLType; import com.metamatrix.common.util.SqlUtil; @@ -304,7 +308,20 @@ currentValue = MMBlob.newInstance(new StreamingLobChunckProducer.Factory(this.statement.getDQP(), this.requestID, (Streamable)currentValue), (BlobType) currentValue); } else if (currentValue instanceof XMLType) { - currentValue = MMSQLXML.newInstance(new StreamingLobChunckProducer.Factory(this.statement.getDQP(), this.requestID, (Streamable)currentValue), (XMLType) currentValue); + XMLType type = (XMLType)currentValue; + if (Boolean.getBoolean(Streamable.FORCE_STREAMING)) { + SQLXML sourceSQLXML = type.getReference(); + if (sourceSQLXML != null) { + return sourceSQLXML; + } + } + final StreamingLobChunckProducer.Factory factory = new StreamingLobChunckProducer.Factory(this.statement.getDQP(), this.requestID, (Streamable)currentValue); + currentValue = new SQLXMLImpl(new InputStreamFactory(Streamable.ENCODING) { + @Override + public InputStream getInputStream() throws IOException { + return new LobChunkInputStream(factory.getLobChunkProducer()); + } + }); } else if (currentValue instanceof java.util.Date) { return TimestampWithTimezone.create((java.util.Date)currentValue, serverTimeZone, getDefaultCalendar(), currentValue.getClass()); Deleted: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java =================================================================== --- trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,131 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.jdbc; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Reader; -import java.io.Writer; -import java.nio.charset.Charset; -import java.sql.SQLException; -//## JDBC4.0-begin ## -import java.sql.SQLXML; -//## JDBC4.0-end ## - -/*## JDBC3.0-JDK1.5-begin ## -import com.metamatrix.core.jdbc.SQLXML; -## JDBC3.0-JDK1.5-end ##*/ - -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.stream.StreamSource; - -import com.metamatrix.common.lob.LobChunkInputStream; -import com.metamatrix.common.types.Streamable; -import com.metamatrix.common.types.XMLType; -import com.metamatrix.common.util.SqlUtil; -import com.metamatrix.dqp.client.impl.StreamingLobChunckProducer; - - -/** - * A MM specific implementation of the SQLXML object which is capable of - * accessing the SQLXML object as local object when used in the embedded product - * at the same time capable of using the streaming interface when used in - * remote clients. - */ -public class MMSQLXML implements SQLXML { - private final StreamingLobChunckProducer.Factory lobChunckFactory; - - public static SQLXML newInstance(StreamingLobChunckProducer.Factory lobChunckFactory, XMLType srcXML) throws SQLException { - if (Boolean.getBoolean(Streamable.FORCE_STREAMING)) { - SQLXML sourceSQLXML = srcXML.getReference(); - if (sourceSQLXML != null) { - return sourceSQLXML; - } - } - return new MMSQLXML(lobChunckFactory); - } - - public MMSQLXML(StreamingLobChunckProducer.Factory lobChunckFactory) throws SQLException { - this.lobChunckFactory = lobChunckFactory; - } - - public Reader getCharacterStream() throws SQLException { - return new LobChunkInputStream(lobChunckFactory.getLobChunkProducer()).getUTF16Reader(); - } - - public String getString() throws SQLException { - LobChunkInputStream in = new LobChunkInputStream(lobChunckFactory.getLobChunkProducer()); - try { - //## JDBC4.0-begin ## - return new String(in.getByteContents(), Charset.forName("UTF-16")); //$NON-NLS-1$ - //## JDBC4.0-end ## - /*## JDBC3.0-JDK1.5-begin ## - return new String(in.getByteContents(), "UTF-16"); //$NON-NLS-1$ - ## JDBC3.0-JDK1.5-end ##*/ - } catch (IOException e) { - throw MMSQLException.create(e); - } - } - - public String toString() { - try { - return getString(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - public <T extends Source> T getSource(Class<T> sourceClass) throws SQLException { - if (sourceClass == null || sourceClass == StreamSource.class) { - return (T)new StreamSource(getCharacterStream()); - } - throw new SQLException(JDBCPlugin.Util.getString("MMSQLXML.unsupported_source", sourceClass)); //$NON-NLS-1$ - } - - public InputStream getBinaryStream() throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); - } - - public void free() throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); - } - - public OutputStream setBinaryStream() throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); - } - - public Writer setCharacterStream() throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); - } - - public <T extends Result> T setResult(Class<T> resultClass) - throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); - } - - public void setString(String value) throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); - } -} Deleted: trunk/common-core/src/main/java/com/metamatrix/common/types/BaseXMLTranslator.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/BaseXMLTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/BaseXMLTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,113 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.types; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.Properties; - -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; - -/** - * Implements some common menthods that can be shared. - */ -public abstract class BaseXMLTranslator implements XMLTranslator { - static final String XMLPI = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; //$NON-NLS-1$ - static final String newLine = "\n"; //$NON-NLS-1$ - - Properties properties; - - protected BaseXMLTranslator(Properties props) { - this.properties = props; - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getString() - */ - public String getString() throws IOException { - try { - Transformer t = TransformerFactory.newInstance().newTransformer(new StreamSource(new StringReader(idenityTransform))); - StringWriter sw = new StringWriter(); - sw.write(XMLPI); - if (useIndentation()) { - sw.write(newLine); - } - t.transform(getSource(), new StreamResult(sw)); - return sw.toString(); - } catch (Exception e) { - e.printStackTrace(); - throw new IOException(e.getMessage()); - } - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getReader() - */ - public Reader getReader() throws IOException { - return new StringReader(getString()); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getInputStream() - */ - public InputStream getInputStream() throws IOException { - return new ByteArrayInputStream(getBytes()); - } - - /** - * Get the XML contents in byte array form. - */ - public byte[] getBytes() throws IOException { - try { - Transformer t = TransformerFactory.newInstance().newTransformer(new StreamSource(new StringReader(idenityTransform))); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - stream.write(XMLPI.getBytes()); - if (useIndentation()) { - stream.write(newLine.getBytes()); - } - t.transform(getSource(), new StreamResult(stream)); - return stream.toByteArray(); - } catch (Exception e) { - throw new IOException(e.getMessage()); - } - } - - public Properties getProperties() { - return this.properties; - } - - private boolean useIndentation() { - if (getProperties() != null) { - return "yes".equalsIgnoreCase(getProperties().getProperty(INDENT)); //$NON-NLS-1$ - } - return false; - } -} Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/DataTypeManager.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/DataTypeManager.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/DataTypeManager.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,6 +22,7 @@ package com.metamatrix.common.types; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Blob; @@ -42,17 +43,15 @@ import java.util.Map; import java.util.Set; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.sax.SAXSource; -import javax.xml.transform.stream.StreamSource; +import javax.xml.transform.Source; import com.metamatrix.common.types.basic.AnyToObjectTransform; import com.metamatrix.common.types.basic.AnyToStringTransform; import com.metamatrix.common.types.basic.BooleanToNumberTransform; -import com.metamatrix.common.types.basic.FloatingNumberToBigDecimalTransform; -import com.metamatrix.common.types.basic.FloatingNumberToBigIntegerTransform; import com.metamatrix.common.types.basic.FixedNumberToBigDecimalTransform; import com.metamatrix.common.types.basic.FixedNumberToBigIntegerTransform; +import com.metamatrix.common.types.basic.FloatingNumberToBigDecimalTransform; +import com.metamatrix.common.types.basic.FloatingNumberToBigIntegerTransform; import com.metamatrix.common.types.basic.NullToAnyTransform; import com.metamatrix.common.types.basic.NumberToBooleanTransform; import com.metamatrix.common.types.basic.NumberToByteTransform; @@ -64,6 +63,7 @@ import com.metamatrix.common.types.basic.ObjectToAnyTransform; import com.metamatrix.core.CorePlugin; import com.metamatrix.core.ErrorMessageKeys; +import com.metamatrix.core.MetaMatrixRuntimeException; import com.metamatrix.core.util.ArgCheck; /** @@ -602,7 +602,7 @@ return new BlobType(value); } }); - sourceConverters.put(byte[].class, new SourceTransform<byte[], BlobType>() { + addSourceTransform(byte[].class, new SourceTransform<byte[], BlobType>() { //## JDBC4.0-begin ## @Override //## JDBC4.0-end ## @@ -610,7 +610,7 @@ return new BlobType(BlobType.createBlob(value)); } }); - sourceConverters.put(SQLXML.class, new SourceTransform<SQLXML, XMLType>() { + addSourceTransform(SQLXML.class, new SourceTransform<SQLXML, XMLType>() { //## JDBC4.0-begin ## @Override //## JDBC4.0-end ## @@ -618,34 +618,27 @@ return new XMLType(value); } }); - sourceConverters.put(DOMSource.class, new SourceTransform<DOMSource, XMLType>() { + //Note: the default transform from non-InputStreamFactory source is a fully materialized string + addSourceTransform(Source.class, new SourceTransform<Source, XMLType>() { //## JDBC4.0-begin ## @Override //## JDBC4.0-end ## - public XMLType transform(DOMSource value) { - return new XMLType(new SQLXMLImpl(value)); + public XMLType transform(Source value) { + if (value instanceof InputStreamFactory) { + return new XMLType(new SQLXMLImpl((InputStreamFactory)value)); + } + StandardXMLTranslator sxt = new StandardXMLTranslator(value, null); + try { + return new XMLType(new SQLXMLImpl(sxt.getString())); + } catch (IOException e) { + throw new MetaMatrixRuntimeException(e); + } } }); - sourceConverters.put(SAXSource.class, new SourceTransform<SAXSource, XMLType>() { + addSourceTransform(Date.class, new SourceTransform<Date, Timestamp>() { //## JDBC4.0-begin ## @Override //## JDBC4.0-end ## - public XMLType transform(SAXSource value) { - return new XMLType(new SQLXMLImpl(value)); - } - }); - sourceConverters.put(StreamSource.class, new SourceTransform<StreamSource, XMLType>() { - //## JDBC4.0-begin ## - @Override - //## JDBC4.0-end ## - public XMLType transform(StreamSource value) { - return new XMLType(new SQLXMLImpl(value)); - } - }); - sourceConverters.put(Date.class, new SourceTransform<Date, Timestamp>() { - //## JDBC4.0-begin ## - @Override - //## JDBC4.0-end ## public Timestamp transform(Date value) { return new Timestamp(value.getTime()); } @@ -702,4 +695,8 @@ || DataTypeManager.DefaultDataTypes.CLOB.equals(type) || DataTypeManager.DefaultDataTypes.XML.equals(type); } + + public static <S> void addSourceTransform(Class<S> sourceClass, SourceTransform<S, ?> transform) { + sourceConverters.put(sourceClass, transform); + } } Added: trunk/common-core/src/main/java/com/metamatrix/common/types/InputStreamFactory.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/InputStreamFactory.java (rev 0) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/InputStreamFactory.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -0,0 +1,62 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.types; + +import java.io.IOException; +import java.io.InputStream; + +import javax.xml.transform.Source; + +public abstract class InputStreamFactory implements Source { + + private String encoding; + private String systemId; + + public InputStreamFactory(String encoding) { + this.encoding = encoding; + } + + /** + * Get a new InputStream + * @return + */ + public abstract InputStream getInputStream() throws IOException; + + public String getEncoding() { + return encoding; + } + + @Override + public String getSystemId() { + return this.systemId; + } + + @Override + public void setSystemId(String systemId) { + this.systemId = systemId; + } + + public void free() throws IOException { + + } +} Property changes on: trunk/common-core/src/main/java/com/metamatrix/common/types/InputStreamFactory.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/SQLXMLImpl.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/SQLXMLImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/SQLXMLImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,138 +22,111 @@ package com.metamatrix.common.types; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; +import java.io.UnsupportedEncodingException; import java.io.Writer; import java.sql.SQLException; -//## JDBC4.0-begin ## import java.sql.SQLXML; -//## JDBC4.0-end ## -/*## JDBC3.0-JDK1.5-begin ## -import com.metamatrix.core.jdbc.SQLXML; -## JDBC3.0-JDK1.5-end ##*/ -import java.util.Properties; - import javax.xml.transform.Result; import javax.xml.transform.Source; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.sax.SAXSource; import javax.xml.transform.stream.StreamSource; import com.metamatrix.common.util.SqlUtil; +import com.metamatrix.core.MetaMatrixRuntimeException; +import com.metamatrix.core.util.ObjectConverterUtil; /** - * This metamatrix specific implementation of the SQLXML interface; + * Default SQLXML impl */ public class SQLXMLImpl implements SQLXML { - XMLTranslator translator; - - public SQLXMLImpl(String str) { - this.translator = new XMLStreamSourceTranslator(str, new Properties()); - } - - public SQLXMLImpl(String str, Properties props) { - this.translator = new XMLStreamSourceTranslator(str, props); - } + private InputStreamFactory streamFactory; - public SQLXMLImpl(char[] str) { - this.translator = new XMLStreamSourceTranslator(str, new Properties()); + public SQLXMLImpl(final byte[] bytes) { + setStreamFactory(bytes); } - public SQLXMLImpl(char[] str, Properties props) { - this.translator = new XMLStreamSourceTranslator(str, props); - } + private void setStreamFactory(final byte[] bytes) { + this.streamFactory = new InputStreamFactory(Streamable.ENCODING) { + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(bytes); + } + }; + } - public SQLXMLImpl(XMLReaderFactory factory) { - this(factory, new Properties()); + public SQLXMLImpl(final String str) { + try { + setStreamFactory(str.getBytes(Streamable.ENCODING)); + } catch (UnsupportedEncodingException e) { + throw new MetaMatrixRuntimeException(e); + } } - - // the reason we have factory insted of reader is the xml may be - // and can be streamed multiple times by a client. If we have reader - // one it closed it may not be accessable. - public SQLXMLImpl(XMLReaderFactory factory, Properties props) { - this.translator = new XMLStreamSourceTranslator(factory, props); - } - - public SQLXMLImpl(StreamSource streamSource) { - this(streamSource, new Properties()); - } - public SQLXMLImpl(StreamSource streamSource, Properties props) { - this.translator = new XMLStreamSourceTranslator(streamSource, props); + public SQLXMLImpl(InputStreamFactory factory) { + this.streamFactory = factory; } - - public SQLXMLImpl(DOMSource domSource) { - this.translator = new XMLDomSourceTranslator(domSource, new Properties()); - } - public SQLXMLImpl(DOMSource domSource, Properties props) { - this.translator = new XMLDomSourceTranslator(domSource, props); - } - - public SQLXMLImpl(SAXSource saxSource) { - this.translator = new XMLSAXSourceTranslator(saxSource, new Properties()); - } - - public SQLXMLImpl(SAXSource saxSource, Properties props) { - this.translator = new XMLSAXSourceTranslator(saxSource, props); - } - - public SQLXMLImpl(XMLTranslator translator) { - this.translator = translator; - } - public <T extends Source> T getSource(Class<T> sourceClass) throws SQLException { - try { - Source c = this.translator.getSource(); - if (sourceClass == null || sourceClass == c.getClass()) { - return (T)c; - } - } catch (IOException e) { - throw new SQLException(e.getMessage()); - } + if (sourceClass == null || sourceClass == StreamSource.class) { + return (T)new StreamSource(getBinaryStream()); + } throw new SQLException("Unsupported source type " + sourceClass); } public Reader getCharacterStream() throws SQLException { - try { - return translator.getReader(); - } catch (IOException e) { - throw new SQLException(e.getMessage()); - } + if (this.streamFactory == null) { + throw new SQLException("SQLXML already freed"); + } + try { + return new InputStreamReader(this.streamFactory.getInputStream(), this.streamFactory.getEncoding()); + } catch (IOException e) { + SQLException ex = new SQLException(e.getMessage()); + ex.initCause(e); + throw ex; + } } public InputStream getBinaryStream() throws SQLException { + if (this.streamFactory == null) { + throw new SQLException("SQLXML already freed"); + } try { - return translator.getInputStream(); - } catch (IOException e) { - throw new SQLException(e.getMessage()); - } + return this.streamFactory.getInputStream(); + } catch (IOException e) { + SQLException ex = new SQLException(e.getMessage()); + ex.initCause(e); + throw ex; + } } public String getString() throws SQLException { try { - return translator.getString(); + return new String(ObjectConverterUtil.convertToByteArray(getBinaryStream()), Streamable.ENCODING); } catch (IOException e) { - throw new SQLException(e.getMessage()); + SQLException ex = new SQLException(e.getMessage()); + ex.initCause(e); + throw ex; } } public OutputStream setBinaryStream() throws SQLException { - throw new SQLException("not implemented");//$NON-NLS-1$ + throw SqlUtil.createFeatureNotSupportedException(); } public Writer setCharacterStream() throws SQLException { - throw new SQLException("not implemented");//$NON-NLS-1$ + throw SqlUtil.createFeatureNotSupportedException(); } public void setString(String value) throws SQLException { - throw new SQLException("not implemented");//$NON-NLS-1$ + throw SqlUtil.createFeatureNotSupportedException(); } public String toString() { @@ -165,7 +138,16 @@ } public void free() throws SQLException { - throw SqlUtil.createFeatureNotSupportedException(); + if (this.streamFactory != null) { + try { + this.streamFactory.free(); + this.streamFactory = null; + } catch (IOException e) { + SQLException ex = new SQLException(e.getMessage()); + ex.initCause(e); + throw ex; + } + } } public <T extends Result> T setResult(Class<T> resultClass) Copied: trunk/common-core/src/main/java/com/metamatrix/common/types/StandardXMLTranslator.java (from rev 1720, trunk/common-core/src/main/java/com/metamatrix/common/types/BaseXMLTranslator.java) =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/StandardXMLTranslator.java (rev 0) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/StandardXMLTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -0,0 +1,85 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.types; + +import java.io.IOException; +import java.io.StringReader; +import java.io.Writer; +import java.util.Properties; + +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; + +public class StandardXMLTranslator extends XMLTranslator { + static String lineSep = System.getProperty("line.separator"); //$NON-NLS-1$ + public static final String INDENT = "indent"; //$NON-NLS-1$ + + public static final String idenityTransform = + "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">" + lineSep + //$NON-NLS-1$ + "<xsl:output method = \"xml\" omit-xml-declaration=\"yes\"/>" + lineSep + //$NON-NLS-1$ + "<xsl:template match=\"@*|node()\">" + lineSep +//$NON-NLS-1$ + " <xsl:copy>\r\n" + lineSep+ //$NON-NLS-1$ + " <xsl:apply-templates select=\"@*|node()\"/>" +lineSep+//$NON-NLS-1$ + " </xsl:copy>" + lineSep + //$NON-NLS-1$ + "</xsl:template>" + lineSep +//$NON-NLS-1$ + "</xsl:stylesheet>" + lineSep + //$NON-NLS-1$ + ""; //$NON-NLS-1$ + + static final String XMLPI = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; //$NON-NLS-1$ + static final String newLine = "\n"; //$NON-NLS-1$ + + private Source source; + private Properties properties; + + public StandardXMLTranslator(Source source, Properties props) { + this.source = source; + this.properties = props; + } + + @Override + public void translate(Writer writer) throws IOException { + try { + Transformer t = TransformerFactory.newInstance().newTransformer(new StreamSource(new StringReader(idenityTransform))); + writer.write(XMLPI); + if (useIndentation()) { + writer.write(newLine); + } + t.transform(source, new StreamResult(writer)); + } catch (TransformerException e) { + IOException ex = new IOException(e.getMessage()); + ex.initCause(e); + throw ex; + } + } + + private boolean useIndentation() { + if (properties != null) { + return "yes".equalsIgnoreCase(properties.getProperty(INDENT)); //$NON-NLS-1$ + } + return false; + } +} Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -33,19 +33,13 @@ * <p>A reference stream id is tuple source id for a Streamble object where the * object is in buffer manager, but the contents will never be written to disk; * this is the ID that client needs to reference to get the chunk of data. - * - * <p>A Persistent stream id is Tuple source id under which the server *may* - * have saved the data to disk in buffer manager. In case of XML it is saved - * however in case of Clobs and Blobs it is not saved yet. This id is used by - * the process worker to in case the reference object has lost its state and we - * need to reinsate the object from the disk. */ public abstract class Streamable<T> implements Serializable { + public static final String ENCODING = "UTF-8"; //$NON-NLS-1$ public static final String FORCE_STREAMING = "FORCE_STREAMING"; //$NON-NLS-1$ public static final int STREAMING_BATCH_SIZE_IN_BYTES = 102400; // 100K private String referenceStreamId; - private String persistenceStreamId; protected transient T reference; public Streamable() { @@ -76,14 +70,6 @@ this.referenceStreamId = id; } - public String getPersistenceStreamId() { - return persistenceStreamId; - } - - public void setPersistenceStreamId(String id) { - this.persistenceStreamId = id; - } - @Override public String toString() { return reference.toString(); @@ -103,8 +89,7 @@ return this.reference.equals(other.reference); } - return this.persistenceStreamId == other.persistenceStreamId - && this.referenceStreamId == other.referenceStreamId; + return this.referenceStreamId == other.referenceStreamId; } } Deleted: trunk/common-core/src/main/java/com/metamatrix/common/types/XMLDomSourceTranslator.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/XMLDomSourceTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/XMLDomSourceTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,51 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.types; - -import java.io.IOException; -import java.util.Properties; - -import javax.xml.transform.Source; -import javax.xml.transform.dom.DOMSource; - - -/** - * This class converts the DOMSource input documents into other types of XML - * documents. - */ -public class XMLDomSourceTranslator extends BaseXMLTranslator { - - DOMSource source; - - public XMLDomSourceTranslator(DOMSource source, Properties props) { - super(props); - this.source = source; - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getSource() - */ - public Source getSource() throws IOException { - return source; - } -} Deleted: trunk/common-core/src/main/java/com/metamatrix/common/types/XMLReaderFactory.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/XMLReaderFactory.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/XMLReaderFactory.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,38 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.types; - -import java.io.IOException; -import java.io.Reader; - - -/** - * A factory interface for generating the readers - */ -public interface XMLReaderFactory { - /** - * Get a reader object - * @return - */ - Reader getReader() throws IOException; -} Deleted: trunk/common-core/src/main/java/com/metamatrix/common/types/XMLSAXSourceTranslator.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/XMLSAXSourceTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/XMLSAXSourceTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,50 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.types; - -import java.io.IOException; -import java.util.Properties; - -import javax.xml.transform.Source; -import javax.xml.transform.sax.SAXSource; - -/** - * This class converts the DOMSource input documents into other types of XML - * documents. - */ -public class XMLSAXSourceTranslator extends BaseXMLTranslator { - - SAXSource source; - - public XMLSAXSourceTranslator(SAXSource source, Properties props) { - super(props); - this.source = source; - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getSource() - */ - public Source getSource() throws IOException { - return source; - } -} Deleted: trunk/common-core/src/main/java/com/metamatrix/common/types/XMLStreamSourceTranslator.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/XMLStreamSourceTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/XMLStreamSourceTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,133 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.types; - -import java.io.CharArrayReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Properties; - -import javax.xml.transform.Source; -import javax.xml.transform.stream.StreamSource; - - -/** - * This class converts the Stream Source XML feed into other types of - * XML Sources. - */ -public class XMLStreamSourceTranslator extends BaseXMLTranslator { - - char[] srcString; - XMLReaderFactory readerFactory; - - public XMLStreamSourceTranslator(String xmlSource, Properties props) { - super(props); - this.srcString = xmlSource.toCharArray(); - this.readerFactory = new CharArrayXMLReader(this.srcString); - } - - public XMLStreamSourceTranslator(char[] xmlSource, Properties props) { - super(props); - this.srcString = xmlSource; - this.readerFactory = new CharArrayXMLReader(this.srcString); - } - - public XMLStreamSourceTranslator(final StreamSource source, Properties props) { - super(props); - this.readerFactory = new StreamSourceXMLReader(source); - } - - public XMLStreamSourceTranslator(XMLReaderFactory factory, Properties props) { - super(props); - this.readerFactory = factory; - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getSource() - */ - public Source getSource() throws IOException { - return new StreamSource(this.readerFactory.getReader()); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getReader() - */ - public Reader getReader() throws IOException { - StreamSource src = (StreamSource)getSource(); - if (src.getReader() != null) { - return src.getReader(); - } - return super.getReader(); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getInputStream() - */ - public InputStream getInputStream() throws IOException { - StreamSource src = (StreamSource)getSource(); - if (src.getInputStream() != null) { - return src.getInputStream(); - } - return super.getInputStream(); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getString() - */ - public String getString() throws IOException { - if (this.srcString != null) { - return new String(this.srcString); - } - return super.getString(); - } - - static class CharArrayXMLReader implements XMLReaderFactory{ - char[] contents; - - public CharArrayXMLReader(char[] content) { - this.contents = content; - } - - public Reader getReader() throws IOException { - return new CharArrayReader(contents); - } - } - - static class StreamSourceXMLReader implements XMLReaderFactory { - StreamSource source; - - public StreamSourceXMLReader(StreamSource source) { - this.source = source; - } - - public Reader getReader() throws IOException { - InputStream stream = source.getInputStream(); - if (stream != null) { - return new InputStreamReader(stream); - } - return source.getReader(); - } - } -} Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/XMLTranslator.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/XMLTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/XMLTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,67 +23,21 @@ package com.metamatrix.common.types; import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.util.Properties; +import java.io.StringWriter; +import java.io.Writer; -import javax.xml.transform.Source; - - /** * This an interface defined to convert the various kinds of the XML sources - * defined into another source kinds. For example a DOMSource.class source can be - * converted to String or SAXSource, or StreamSource is converted to DOMSource etc. + * defined into a character stream. */ -public interface XMLTranslator { - static String lineSep = System.getProperty("line.separator"); //$NON-NLS-1$ - public static final String INDENT = "indent"; //$NON-NLS-1$ +public abstract class XMLTranslator { - public static final String idenityTransform = - "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">" + lineSep + //$NON-NLS-1$ - "<xsl:output method = \"xml\" omit-xml-declaration=\"yes\"/>" + lineSep + //$NON-NLS-1$ - "<xsl:template match=\"@*|node()\">" + lineSep +//$NON-NLS-1$ - " <xsl:copy>\r\n" + lineSep+ //$NON-NLS-1$ - " <xsl:apply-templates select=\"@*|node()\"/>" +lineSep+//$NON-NLS-1$ - " </xsl:copy>" + lineSep + //$NON-NLS-1$ - "</xsl:template>" + lineSep +//$NON-NLS-1$ - "</xsl:stylesheet>" + lineSep + //$NON-NLS-1$ - ""; //$NON-NLS-1$ + public abstract void translate(Writer writer) throws IOException; - /** - * Get String form of the XML - * @return string representing the XML source - * @throws IOException - */ - String getString() throws IOException; + public String getString() throws IOException { + StringWriter writer = new StringWriter(); + this.translate(writer); + return writer.toString(); + } - /** - * Get the XML in the original source form; with however it got created. - * @return - * @throws IOException - */ - public Source getSource() throws IOException; - - - /** - * Get a Reader for the XML contents; - * @return a Reader object for streaming the XML contents. - * @throws IOException - */ - public Reader getReader() throws IOException; - - /** - * Get a InputStream for the XMl contents; - * @return a InputStream object for streaming the XML contents. - * @throws IOException - */ - public InputStream getInputStream() throws IOException; - - /** - * Any specific Properties needed by the translator process, such - * as indenting etc. - * @return properties - */ - public Properties getProperties(); - } Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/basic/StringToSQLXMLTransform.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/basic/StringToSQLXMLTransform.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/basic/StringToSQLXMLTransform.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -31,8 +31,9 @@ import org.w3c.dom.Document; import org.xml.sax.InputSource; +import com.metamatrix.common.types.SQLXMLImpl; +import com.metamatrix.common.types.StandardXMLTranslator; import com.metamatrix.common.types.Transform; -import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.TransformationException; import com.metamatrix.common.types.XMLType; import com.metamatrix.core.CorePlugin; @@ -53,7 +54,8 @@ try{ DocumentBuilder parser = factory.newDocumentBuilder(); Document doc = parser.parse(new InputSource(new StringReader(xml))); - return new XMLType(new SQLXMLImpl(new DOMSource(doc))); + StandardXMLTranslator sxt = new StandardXMLTranslator(new DOMSource(doc), null); + return new XMLType(new SQLXMLImpl(sxt.getString())); } catch (Exception e){ throw new TransformationException(e, CorePlugin.Util.getString("invalid_string")); //$NON-NLS-1$ Modified: trunk/common-core/src/test/java/com/metamatrix/common/types/TestSQLXMLImpl.java =================================================================== --- trunk/common-core/src/test/java/com/metamatrix/common/types/TestSQLXMLImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/test/java/com/metamatrix/common/types/TestSQLXMLImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,15 +22,10 @@ package com.metamatrix.common.types; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.Reader; -import java.io.StringReader; import java.sql.SQLException; -import java.util.Properties; -import javax.xml.transform.Source; import javax.xml.transform.stream.StreamSource; import junit.framework.TestCase; @@ -44,57 +39,34 @@ public class TestSQLXMLImpl extends TestCase { String testStr = "<foo>test</foo>"; //$NON-NLS-1$ - - XMLTranslator translator = new XMLTranslator() { - public String getString() throws IOException { - return testStr; - } - public Source getSource() throws IOException { - return new StreamSource(new StringReader(testStr)); - } - public Reader getReader() throws IOException { - return new StringReader(testStr); - } - - public InputStream getInputStream() throws IOException { - return new ByteArrayInputStream(testStr.getBytes()); - } - - public Properties getProperties() { - Properties p = new Properties(); - p.setProperty("indent", "yes"); //$NON-NLS-1$ //$NON-NLS-2$ - return p; - } - }; - + //## JDBC4.0-begin ## public void testGetSource() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); + SQLXMLImpl xml = new SQLXMLImpl(testStr); assertTrue(xml.getSource(null) instanceof StreamSource); StreamSource ss = (StreamSource)xml.getSource(null); - assertEquals(testStr, getContents(ss.getReader())); + assertEquals(testStr, new String(ObjectConverterUtil.convertToByteArray(ss.getInputStream()), Streamable.ENCODING)); } //## JDBC4.0-end ## public void testGetCharacterStream() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); + SQLXMLImpl xml = new SQLXMLImpl(testStr); assertEquals(testStr, getContents(xml.getCharacterStream())); } public void testGetBinaryStream() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); - assertEquals(testStr, ObjectConverterUtil.convertToString(xml.getBinaryStream())); - + SQLXMLImpl xml = new SQLXMLImpl(testStr); + assertEquals(testStr, new String(ObjectConverterUtil.convertToByteArray(xml.getBinaryStream()), Streamable.ENCODING)); } public void testGetString() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); + SQLXMLImpl xml = new SQLXMLImpl(testStr); assertEquals(testStr, xml.getString()); } public void testSetBinaryStream() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); + SQLXMLImpl xml = new SQLXMLImpl(testStr); try { xml.setBinaryStream(); fail("we do not support this yet.."); //$NON-NLS-1$ @@ -103,7 +75,7 @@ } public void testSetCharacterStream() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); + SQLXMLImpl xml = new SQLXMLImpl(testStr); try { xml.setCharacterStream(); fail("we do not support this yet.."); //$NON-NLS-1$ @@ -112,7 +84,7 @@ } public void testSetString() throws Exception { - SQLXMLImpl xml = new SQLXMLImpl(translator); + SQLXMLImpl xml = new SQLXMLImpl(testStr); try { xml.setString(testStr); fail("we do not support this yet.."); //$NON-NLS-1$ Modified: trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLStreamSourceTranslator.java =================================================================== --- trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLStreamSourceTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLStreamSourceTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,19 +22,12 @@ package com.metamatrix.common.types; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; import java.io.StringReader; import java.util.Properties; import java.util.StringTokenizer; import javax.xml.transform.stream.StreamSource; -import com.metamatrix.core.util.ObjectConverterUtil; - import junit.framework.TestCase; @@ -83,83 +76,12 @@ " </publishingInformation>\r\n" + //$NON-NLS-1$ " </book>\r\n" + //$NON-NLS-1$ "</Books:bookCollection>"; //$NON-NLS-1$ - /* - * Test method for 'com.metamatrix.common.types.XMLStreamSourceTranslator.getReader()' - */ - public void testGetReader() throws Exception { - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(new StreamSource(new StringReader(sourceXML)), new Properties()); - Reader reader = translator.getReader(); - assertEquals(sourceXML, ObjectConverterUtil.convertToString(reader)); - } - - public void testCharInput() throws Exception { - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(sourceXML.toCharArray(), new Properties()); - Reader reader = translator.getReader(); - assertEquals(sourceXML, ObjectConverterUtil.convertToString(reader)); - } - - public void testXMLReaderFactory() throws Exception { - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(new XMLReaderFactory() { - public Reader getReader() throws IOException { - return new StringReader(sourceXML); - } - }, new Properties()); - Reader reader = translator.getReader(); - assertEquals(sourceXML, ObjectConverterUtil.convertToString(reader)); - } - - /* - * Test method for 'com.metamatrix.common.types.XMLStreamSourceTranslator.getInputStream()' - */ - public void testGetInputStream() throws Exception { - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(new StreamSource(new ByteArrayInputStream(sourceXML.getBytes())), new Properties()); - InputStream stream = translator.getInputStream(); - ByteArrayOutputStream out = getCoontents(stream); - compareDocuments(sourceXML, new String(out.toByteArray())); - } - - private ByteArrayOutputStream getCoontents(InputStream stream) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - - int chr = stream.read(); - while(chr != -1) { - out.write((byte)chr); - chr = stream.read(); - } - return out; - } - + public void testStreamSourceWithStream() throws Exception { - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(new StreamSource(new StringReader(sourceXML)), new Properties()); - InputStream stream = translator.getInputStream(); - ByteArrayOutputStream out = getCoontents(stream); - compareDocuments(sourceXML, new String(out.toByteArray())); + StandardXMLTranslator translator = new StandardXMLTranslator(new StreamSource(new StringReader(sourceXML)), new Properties()); + compareDocuments(sourceXML, translator.getString()); } - /* - * Test method for 'com.metamatrix.common.types.XMLStreamSourceTranslator.getSource()' - */ - public void testGetSource() throws Exception { - StreamSource src = new StreamSource(new StringReader(sourceXML)); - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(src, new Properties()); - assertTrue(translator.getSource() instanceof StreamSource); - StreamSource s = (StreamSource)translator.getSource(); - Reader reader = s.getReader(); - assertEquals(sourceXML, ObjectConverterUtil.convertToString(reader)); - } - - /* - * Test method for 'com.metamatrix.common.types.BaseXMLTranslator.getString()' - */ - public void testGetString() throws Exception { - XMLStreamSourceTranslator translator = new XMLStreamSourceTranslator(sourceXML, new Properties()); - assertEquals(sourceXML, translator.getString()); - - // in the case of the stream source there is optimization that - // if a string is used to xmlize then same string is returned. - assertEquals(sourceXML, translator.getString()); - } - private void compareDocuments(String expectedDoc, String actualDoc) { StringTokenizer tokens1 = new StringTokenizer(expectedDoc, "\r\n"); //$NON-NLS-1$ StringTokenizer tokens2 = new StringTokenizer(actualDoc, "\n");//$NON-NLS-1$ Modified: trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java =================================================================== --- trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -52,7 +52,6 @@ String pkey = "peresistkeytodata"; //$NON-NLS-1$ XMLType xv = new XMLType(xml); xv.setReferenceStreamId(key); - xv.setPersistenceStreamId(pkey); // now force to serialize File saved = new File(UnitTestUtil.getTestScratchPath()+"/xmlsaved.bin"); //$NON-NLS-1$ @@ -66,7 +65,6 @@ // make sure we have kept the reference stream id assertEquals(key, read.getReferenceStreamId()); - assertEquals(pkey, read.getPersistenceStreamId()); // and lost the original object assertNull(read.getReference()); Deleted: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/CachingConnector.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/CachingConnector.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/CachingConnector.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,28 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.connector.xml; - - -public interface CachingConnector extends StatefulConnector { - -} \ No newline at end of file Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnection.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnection.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnection.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -32,7 +32,7 @@ public XMLConnectorState getState(); - public CachingConnector getConnector(); + public StatefulConnector getConnector(); public TrustedPayloadHandler getTrustedPayloadHandler() throws ConnectorException; Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnectorState.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnectorState.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/XMLConnectorState.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,6 +23,9 @@ package com.metamatrix.connector.xml; +import java.sql.SQLXML; +import java.util.Map; + import org.teiid.connector.api.Connection; import org.teiid.connector.api.ConnectorCapabilities; import org.teiid.connector.api.ConnectorEnvironment; @@ -43,12 +46,14 @@ public abstract IQueryPreprocessor getPreprocessor(); - public abstract Connection getConnection(CachingConnector connector, + public abstract Connection getConnection(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException; public String getPluggableInputStreamFilterClass(); public boolean isCaching(); + + public Map<String, SQLXML> getResponses(); } \ No newline at end of file Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/AbstractCachingConnector.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/AbstractCachingConnector.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/AbstractCachingConnector.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -25,13 +25,10 @@ import org.teiid.connector.api.ConnectorEnvironment; import org.teiid.connector.api.ConnectorException; -import com.metamatrix.connector.xml.CachingConnector; - //TODO This class used to hold the internal connector cache. // I'm leaving it here so I can add cachein at a scope other than REQUEST. -public abstract class AbstractCachingConnector extends LoggingConnector - implements CachingConnector { +public abstract class AbstractCachingConnector extends LoggingConnector { public AbstractCachingConnector() { super(); Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectionImpl.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectionImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectionImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -31,15 +31,15 @@ import org.teiid.connector.api.ExecutionContext; import org.teiid.connector.basic.BasicConnection; -import com.metamatrix.connector.xml.CachingConnector; import com.metamatrix.connector.xml.SecureConnectorState; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.TrustedPayloadHandler; import com.metamatrix.connector.xml.XMLConnection; import com.metamatrix.connector.xml.XMLConnectorState; public class XMLConnectionImpl extends BasicConnection implements XMLConnection { - private CachingConnector connector; + private StatefulConnector connector; private ConnectorEnvironment connectorEnv; @@ -53,7 +53,7 @@ private TrustedPayloadHandler payloadHandler; - public XMLConnectionImpl(CachingConnector connector, ExecutionContext context, + public XMLConnectionImpl(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment connectorEnv) throws ConnectorException { this.connector = connector; this.connectorEnv = connectorEnv; @@ -81,7 +81,7 @@ return connector.getState(); } - public CachingConnector getConnector() { + public StatefulConnector getConnector() { return connector; } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectorStateImpl.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectorStateImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/base/XMLConnectorStateImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,6 +23,10 @@ package com.metamatrix.connector.xml.base; +import java.io.InputStream; +import java.sql.SQLXML; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.teiid.connector.api.ConnectorCapabilities; @@ -33,6 +37,7 @@ import com.metamatrix.connector.xml.IQueryPreprocessor; import com.metamatrix.connector.xml.SAXFilterProvider; import com.metamatrix.connector.xml.XMLConnectorState; +import com.metamatrix.core.util.ReflectionHelper; public abstract class XMLConnectorStateImpl implements Cloneable, XMLConnectorState { @@ -76,6 +81,8 @@ private String capabilitiesClass; private boolean caching = false; + + private Map<String, SQLXML> responses = new HashMap<String, SQLXML>(); public XMLConnectorStateImpl() { setPreprocess(true); @@ -318,4 +325,29 @@ public String getPluggableInputStreamFilterClass() { return m_pluggableInputStreamFilterClass; } + + public InputStream addStreamFilters(InputStream stream) + throws ConnectorException { + + if (isLogRequestResponse()) { + stream = new LoggingInputStreamFilter(stream, logger); + } + + if (getPluggableInputStreamFilterClass() != null) { + try { + stream = (InputStream) ReflectionHelper.create(getPluggableInputStreamFilterClass(), new Object[] { stream, + logger}, new Class[] { + java.io.InputStream.class, + ConnectorLogger.class }, Thread.currentThread().getContextClassLoader()); + } catch (Exception cnf) { + throw new ConnectorException(cnf); + } + } + return stream; + } + + @Override + public Map<String, SQLXML> getResponses() { + return this.responses; + } } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectionImpl.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectionImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectionImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -8,12 +8,12 @@ import org.teiid.connector.language.IQueryCommand; import org.teiid.connector.metadata.runtime.RuntimeMetadata; -import com.metamatrix.connector.xml.CachingConnector; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.base.XMLConnectionImpl; public class FileConnectionImpl extends XMLConnectionImpl { - public FileConnectionImpl(CachingConnector connector, ExecutionContext context, + public FileConnectionImpl(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment connectorEnv) throws ConnectorException { super(connector,context, connectorEnv); } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectorState.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectorState.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileConnectorState.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,20 +23,15 @@ package com.metamatrix.connector.xml.file; -import java.io.InputStream; -import java.lang.reflect.Constructor; import java.util.Properties; import org.teiid.connector.api.Connection; import org.teiid.connector.api.ConnectorEnvironment; import org.teiid.connector.api.ConnectorException; -import org.teiid.connector.api.ConnectorLogger; import org.teiid.connector.api.ExecutionContext; -import com.metamatrix.connector.xml.CachingConnector; -import com.metamatrix.connector.xml.base.LoggingInputStreamFilter; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.base.XMLConnectorStateImpl; -import com.metamatrix.connector.xml.cache.CachingInputStreamFilter; public class FileConnectorState extends XMLConnectorStateImpl { @@ -105,34 +100,8 @@ return m_directoryPath; } - public Connection getConnection(CachingConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException { + public Connection getConnection(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException { return new FileConnectionImpl(connector, context, environment); } - - public InputStream addStreamFilters(InputStream stream) - throws ConnectorException { - - if (isLogRequestResponse()) { - stream = new LoggingInputStreamFilter(stream, logger); - } - - try { - Class pluggableFilter = Class - .forName(getPluggableInputStreamFilterClass()); - Constructor ctor = pluggableFilter.getConstructor(new Class[] { - java.io.InputStream.class, - ConnectorLogger.class }); - stream = (InputStream) ctor.newInstance(new Object[] { stream, - logger }); - } catch (Exception cnf) { - throw new ConnectorException(cnf); - } - return stream; - } - public InputStream addCachingStreamFilters(InputStream stream, ExecutionContext context, String queryId) throws ConnectorException { - stream = addStreamFilters(stream); - stream = new CachingInputStreamFilter(stream, context, queryId); - return stream; - } } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileExecution.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileExecution.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/file/FileExecution.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,5 +1,6 @@ package com.metamatrix.connector.xml.file; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -24,7 +25,6 @@ import com.metamatrix.connector.xml.base.OutputXPathDesc; import com.metamatrix.connector.xml.base.QueryAnalyzer; import com.metamatrix.connector.xml.base.XMLConnectionImpl; -import com.metamatrix.connector.xml.cache.CachedXMLStream; import com.metamatrix.connector.xml.streaming.BaseStreamingExecution; import com.metamatrix.connector.xml.streaming.DocumentImpl; import com.metamatrix.connector.xml.streaming.InvalidPathException; @@ -241,24 +241,9 @@ } private Document getDocument() throws ConnectorException { - Document doc; - String cacheKey = queryID + Integer.valueOf(docNumber).toString(); - if(state.isCaching()) { - if(null != exeContext.get(queryID)) { - InputStream stream = new CachedXMLStream(exeContext, queryID); - doc = new DocumentImpl(stream, cacheKey); - logger.logTrace("Got " + queryID + " from the cache"); - } else { - InputStream stream = getDocumentStream(docNumber); - stream = state.addCachingStreamFilters(stream, exeContext, queryID); - doc = new DocumentImpl(stream, cacheKey); - } - } else { - InputStream stream = getDocumentStream(docNumber); - stream = state.addStreamFilters(stream); - doc = new DocumentImpl(stream, cacheKey); - } - return doc; + InputStream stream = getDocumentStream(docNumber); + stream = state.addStreamFilters(stream); + return new DocumentImpl(stream, queryID + docNumber); } private InputStream getDocumentStream(int i) throws ConnectorException { @@ -267,7 +252,7 @@ File xmlFile = new File(xmlFileName); logger.logTrace( "XML Connector Framework: retrieving document from " + xmlFileName); //$NON-NLS-1$ - InputStream retval = new FileInputStream(xmlFile); + InputStream retval = new BufferedInputStream(new FileInputStream(xmlFile)); logger.logTrace( "XML Connector Framework: retrieved file " + xmlFileName); //$NON-NLS-1$ return retval; Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectionImpl.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectionImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectionImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -8,12 +8,12 @@ import org.teiid.connector.language.IQueryCommand; import org.teiid.connector.metadata.runtime.RuntimeMetadata; -import com.metamatrix.connector.xml.CachingConnector; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.base.XMLConnectionImpl; public class HTTPConnectionImpl extends XMLConnectionImpl { - public HTTPConnectionImpl(CachingConnector connector, ExecutionContext context, + public HTTPConnectionImpl(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment connectorEnv) throws ConnectorException { super(connector,context, connectorEnv); } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectorState.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectorState.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPConnectorState.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -42,7 +42,7 @@ import org.teiid.connector.api.ConnectorException; import org.teiid.connector.api.ExecutionContext; -import com.metamatrix.connector.xml.CachingConnector; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.base.SecureConnectorStateImpl; public class HTTPConnectorState extends SecureConnectorStateImpl { @@ -407,7 +407,7 @@ return m_httpBasicAuthPwd; } - public Connection getConnection(CachingConnector connector, + public Connection getConnection(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException { return new HTTPConnectionImpl(connector, context, environment); Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPRequest.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPRequest.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/http/HTTPRequest.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -3,15 +3,19 @@ import java.io.IOException; import java.io.InputStream; import java.io.Serializable; -import java.lang.reflect.Constructor; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; +import java.sql.SQLException; +import java.sql.SQLXML; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Properties; +import javax.xml.transform.Source; +import javax.xml.transform.stream.StreamSource; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.NameValuePair; @@ -23,7 +27,6 @@ import org.jdom.Document; import org.teiid.connector.api.ConnectorException; import org.teiid.connector.api.ConnectorLogger; -import org.teiid.connector.api.ExecutionContext; import com.metamatrix.connector.xml.Constants; import com.metamatrix.connector.xml.SAXFilterProvider; @@ -32,12 +35,11 @@ import com.metamatrix.connector.xml.base.CriteriaDesc; import com.metamatrix.connector.xml.base.DocumentBuilder; import com.metamatrix.connector.xml.base.ExecutionInfo; -import com.metamatrix.connector.xml.base.LoggingInputStreamFilter; import com.metamatrix.connector.xml.base.OutputXPathDesc; import com.metamatrix.connector.xml.base.ParameterDescriptor; import com.metamatrix.connector.xml.base.RequestGenerator; -import com.metamatrix.connector.xml.cache.CachedXMLStream; import com.metamatrix.connector.xml.streaming.DocumentImpl; +import com.metamatrix.core.util.Assertion; public class HTTPRequest extends BaseRequest { @@ -50,8 +52,9 @@ public static final String PARM_INPUT_XPATH_TABLE_PROPERTY_NAME = "XPathRootForInput"; //$NON-NLS-1$ public static final String PARM_INPUT_NAMESPACE_TABLE_PROPERTY_NAME = "NamespaceForDocument"; //$NON-NLS-1$ - - + + protected SQLXML response; + public HTTPRequest(HTTPConnectorState state, XMLExecution execution, ExecutionInfo exeInfo, List<CriteriaDesc> parameters) throws ConnectorException { super(state, execution, exeInfo, parameters); @@ -76,35 +79,32 @@ public com.metamatrix.connector.xml.Document getDocumentStream() throws ConnectorException { com.metamatrix.connector.xml.Document document; - ExecutionContext exeContext = execution.getExeContext(); - String cacheKey = getCacheKey(); + //ExecutionContext exeContext = execution.getExeContext(); - // Is this a request part joining across a document - CriteriaDesc criterion = this.exeInfo.getResponseIDCriterion(); - if (null != criterion) { - String responseid = (String) (criterion.getValues().get(0)); - - if(null == exeContext.get(responseid)) { - throw new ConnectorException(Messages.getString("HTTPExecutor.No.doc.in.cache")); - } else { - InputStream stream = new CachedXMLStream(exeContext, responseid); - document = new DocumentImpl(stream, cacheKey); - - } - } else { - // Not a join, but might still be cached. - if (null == exeContext.get(cacheKey)) { - // Not cached, so make the request - SAXFilterProvider provider = null; - provider = getState().getSAXFilterProvider(); - InputStream responseBody = executeRequest(); - InputStream filteredStream = addStreamFilters(responseBody, getLogger()); - document = new DocumentImpl(filteredStream, cacheKey); - } else { - InputStream stream = new CachedXMLStream(exeContext, cacheKey); - document = new DocumentImpl(stream, cacheKey); - } - } + try { + // Is this a request part joining across a document + CriteriaDesc criterion = this.exeInfo.getResponseIDCriterion(); + if (null != criterion) { + String responseid = (String) (criterion.getValues().get(0)); + SQLXML xml = this.execution.getConnection().getState().getResponses().get(responseid); + Assertion.isNotNull(xml); + document = new DocumentImpl(xml.getBinaryStream(), responseid); + } else { + // Not a join, but might still be cached. + //if (null == exeContext.get(cacheKey)) { + // Not cached, so make the request + SAXFilterProvider provider = getState().getSAXFilterProvider(); + InputStream responseBody = executeRequest(); + InputStream filteredStream = getState().addStreamFilters(responseBody); + document = new DocumentImpl(filteredStream, this.execution.getExeContext().getExecutionCountIdentifier()); + //} else { + // InputStream stream = new CachedXMLStream(exeContext, cacheKey); + // document = new DocumentImpl(stream, cacheKey); + //} + } + } catch (SQLException e) { + throw new ConnectorException(e); + } return document; } @@ -137,50 +137,20 @@ return m_allowHttp500; } - - public void release() { - request.releaseConnection(); + if (request != null) { + request.releaseConnection(); + } + if (response != null) { + this.execution.getConnection().getState().getResponses().remove(this.execution.getExeContext().getExecutionCountIdentifier()); + try { + response.free(); + } catch (SQLException e) { + } + response = null; + } } - protected String getCacheKey() throws ConnectorException { - - if (request == null) { - String message = com.metamatrix.connector.xml.http.Messages - .getString("HttpExecutor.cannot.create.cachekey"); - throw new ConnectorException(message); - } - // the key consists of a String in the form of - // |uri|parameterList| - String userName = execution.getConnection().getUser(); - String session = execution.getConnection().getQueryId(); - - StringBuffer cacheKey = new StringBuffer(); - cacheKey.append("|"); //$NON-NLS-1$ - cacheKey.append(userName); - cacheKey.append("|"); - cacheKey.append(session); - cacheKey.append("|"); - cacheKey.append(getUriString()); - cacheKey.append("|"); //$NON-NLS-1$ - - if (request instanceof PostMethod) { - NameValuePair[] pairs = ((PostMethod) request).getParameters(); - if (pairs == null || pairs.length == 0) { - if (((PostMethod) request).getRequestEntity() != null) { - String requestBodyAsString = ((StringRequestEntity) (((PostMethod) request) - .getRequestEntity())).getContent(); - cacheKey.append(requestBodyAsString); - } - } else { - cacheKey.append(generatePairString(pairs)); - } - } else { - cacheKey.append(request.getQueryString()); - } - return cacheKey.toString(); - } - protected InputStream executeRequest() throws ConnectorException { HttpClient client = (getState()).getClient(); XMLConnection conn = execution.getConnection(); @@ -202,10 +172,19 @@ modifyRequest(client, request); InputStream responseBody = m_requestor.fetchXMLDocument(client, request, getAllowHttp500()); - return responseBody; + try { + createSQLXML(new StreamSource(responseBody)); + return response.getBinaryStream(); + } catch (SQLException e) { + throw new ConnectorException(e); + } } + + protected void createSQLXML(Source output) { + response = (SQLXML)this.execution.getConnection().getConnectorEnv().getTypeFacility().convertToRuntimeType(output); + this.execution.getConnection().getState().getResponses().put(this.execution.getExeContext().getExecutionCountIdentifier(), response); + } - private void createRequests() throws ConnectorException { if (checkIfRequestIsNeeded(exeInfo)) { setRequests(this.parameters, getUriString()); @@ -497,26 +476,6 @@ return getState().getLogger(); } - - public InputStream addStreamFilters(InputStream response, ConnectorLogger logger) - throws ConnectorException { - - if(getState().isLogRequestResponse()) { - response = new LoggingInputStreamFilter(response, logger); - } - - InputStream filter = null; - try { - Class pluggableFilter = Thread.currentThread().getContextClassLoader().loadClass(getState().getPluggableInputStreamFilterClass()); - Constructor ctor = pluggableFilter.getConstructor( - new Class[] { java.io.InputStream.class, org.teiid.connector.api.ConnectorLogger.class}); - filter = (InputStream) ctor.newInstance(new Object[] {response, logger}); - } catch (Exception cnf) { - throw new ConnectorException(cnf); - } - return filter; - } - /** * Examines the Query to determine if a request to a source is needed. If any of the * request parameters is a ResponseIn, then we don't need to make a request because it Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectionImpl.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectionImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectionImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -8,12 +8,12 @@ import org.teiid.connector.language.IQueryCommand; import org.teiid.connector.metadata.runtime.RuntimeMetadata; -import com.metamatrix.connector.xml.CachingConnector; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.base.XMLConnectionImpl; public class SOAPConnectionImpl extends XMLConnectionImpl { - public SOAPConnectionImpl(CachingConnector connector, + public SOAPConnectionImpl(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment connectorEnv) throws ConnectorException { super(connector, context, connectorEnv); Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectorState.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectorState.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPConnectorState.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -30,7 +30,7 @@ import org.teiid.connector.api.ConnectorLogger; import org.teiid.connector.api.ExecutionContext; -import com.metamatrix.connector.xml.CachingConnector; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.http.HTTPConnectorState; /** @@ -83,7 +83,7 @@ return soapState.isExceptionOnFault(); } - public Connection getConnection(CachingConnector connector, + public Connection getConnection(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException { return new SOAPConnectionImpl(connector, context, environment); Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPRequest.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPRequest.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/soap/SOAPRequest.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -7,9 +7,6 @@ import javax.xml.namespace.QName; import javax.xml.transform.Source; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import javax.xml.ws.Dispatch; import javax.xml.ws.Service; @@ -28,7 +25,6 @@ import com.metamatrix.connector.xml.base.DocumentBuilder; import com.metamatrix.connector.xml.base.ExecutionInfo; import com.metamatrix.connector.xml.base.RequestGenerator; -import com.metamatrix.connector.xml.cache.CachingOutputStream; import com.metamatrix.connector.xml.http.HTTPConnectorState; import com.metamatrix.connector.xmlsource.soap.SecurityToken; @@ -86,13 +82,8 @@ Source input = new StreamSource(reader); // Invoke the operation. Source output = dispatch.invoke(input); - - // Process the response. - CachingOutputStream out = new CachingOutputStream(execution.getExeContext(), getCacheKey()); - StreamResult result = new StreamResult(out); - Transformer trans = TransformerFactory.newInstance().newTransformer(); - trans.transform(output, result); - return out.getCachedXMLStream(); + createSQLXML(output); + return response.getBinaryStream(); } catch (Exception e) { throw new ConnectorException(e); } @@ -117,18 +108,6 @@ doc = builder.createXMLRequestDoc(bodyParams, (SOAPConnectorState)state, namespacePrefixes, inputParmsXPath); } - protected String getCacheKey() throws ConnectorException { - StringBuffer cacheKey = new StringBuffer(); - cacheKey.append("|"); //$NON-NLS-1$ - cacheKey.append(execution.getConnection().getUser()); - cacheKey.append("|"); - cacheKey.append(execution.getConnection().getQueryId()); - cacheKey.append("|"); - cacheKey.append(getUriString()); - cacheKey.append(requestNumber); - return cacheKey.toString(); - } - public int getDocumentCount() throws ConnectorException { return 1; } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/streaming/BaseStreamingExecution.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/streaming/BaseStreamingExecution.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xml/streaming/BaseStreamingExecution.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -26,14 +26,14 @@ protected StreamingResultsProducer rowProducer; - protected List<Object[]> results; + protected List<Object[]> results = new ArrayList<Object[]>(); private int resultIndex; private Iterator<Document> streamIter; // A single query can result in multiple requests to an XML source. // Each request is abstracted by a result producer. // TODO: Do I really need more than one of theses? - protected List<ResultProducer> resultProducers; + protected List<ResultProducer> resultProducers = new ArrayList<ResultProducer>(); private Iterator<ResultProducer> producerIter; protected ExecutionInfo exeInfo; @@ -53,8 +53,6 @@ exeContext = context; connEnv = env; logger = connection.getConnectorEnv().getLogger(); - resultProducers = new ArrayList<ResultProducer>(); - results = new ArrayList<Object[]>(); } public void cancel() throws ConnectorException { @@ -62,7 +60,9 @@ } public void close() throws ConnectorException { - // nothing to do + for (ResultProducer resultProducer : resultProducers) { + resultProducer.closeStreams(); + } } /** Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/XMLSourceExecution.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/XMLSourceExecution.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/XMLSourceExecution.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -45,6 +45,7 @@ // Connector environment protected ConnectorEnvironment env; private boolean returnedResult; + protected Source returnValue; /** * ctor @@ -65,14 +66,12 @@ } return (SQLXML)result; } - - protected abstract Source getReturnValue(); @Override public List<?> next() throws ConnectorException, DataNotAvailableException { if (!returnedResult) { returnedResult = true; - return Arrays.asList(convertToXMLType(getReturnValue())); + return Arrays.asList(convertToXMLType(returnValue)); } return null; } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/file/FileExecution.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/file/FileExecution.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/file/FileExecution.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,15 +22,13 @@ package com.metamatrix.connector.xmlsource.file; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.InputStream; import java.util.Properties; -import javax.xml.transform.Source; -import javax.xml.transform.stream.StreamSource; - import org.teiid.connector.api.ConnectorEnvironment; import org.teiid.connector.api.ConnectorException; import org.teiid.connector.api.ExecutionContext; @@ -38,6 +36,7 @@ import org.teiid.connector.metadata.runtime.MetadataObject; import org.teiid.connector.metadata.runtime.RuntimeMetadata; +import com.metamatrix.common.types.InputStreamFactory; import com.metamatrix.connector.xmlsource.XMLSourceExecution; import com.metamatrix.connector.xmlsource.XMLSourcePlugin; @@ -50,7 +49,6 @@ private static final String ISO8859 = "ISO-8859-1"; //$NON-NLS-1$ RuntimeMetadata metadata = null; - Source returnValue = null; ExecutionContext context; File rootFolder; private IProcedure procedure; @@ -88,7 +86,7 @@ } // try to open the file and read. - File xmlFile = new File(this.rootFolder, fileName); + final File xmlFile = new File(this.rootFolder, fileName); if (!xmlFile.exists()) { throw new ConnectorException(XMLSourcePlugin.Util.getString("XML_file_not_found", new Object[] {fileName, this.rootFolder.getAbsolutePath()})); //$NON-NLS-1$ } @@ -96,18 +94,15 @@ Properties props = this.env.getProperties(); String encoding = props.getProperty(ENCODING, ISO8859); - try { - this.returnValue = new StreamSource(new InputStreamReader(new FileInputStream(xmlFile), encoding)); - } catch (IOException e) { - throw new ConnectorException(e); - } - + returnValue = new InputStreamFactory(encoding) { + + @Override + public InputStream getInputStream() throws IOException { + return new BufferedInputStream(new FileInputStream(xmlFile)); + } + }; + XMLSourcePlugin.logDetail(this.env.getLogger(), "executing_procedure", new Object[] {procedure.getProcedureName()}); //$NON-NLS-1$ } - @Override - public Source getReturnValue() { - return returnValue; - } - } Modified: trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/soap/SoapExecution.java =================================================================== --- trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/soap/SoapExecution.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/main/java/com/metamatrix/connector/xmlsource/soap/SoapExecution.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -52,7 +52,6 @@ // Connection object. SoapConnection connection; Map outputValues = null; - Source returnValue = null; RuntimeMetadata metadata = null; ExecutionContext context; private IProcedure procedure; @@ -121,9 +120,4 @@ } } - @Override - public Source getReturnValue() { - return returnValue; - } - } Modified: trunk/connectors/connector-xml/src/test/java/com/metamatrix/connector/xml/base/TestXMLConnectorState.java =================================================================== --- trunk/connectors/connector-xml/src/test/java/com/metamatrix/connector/xml/base/TestXMLConnectorState.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/connectors/connector-xml/src/test/java/com/metamatrix/connector/xml/base/TestXMLConnectorState.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -33,7 +33,7 @@ import com.metamatrix.cdk.api.EnvironmentUtility; import com.metamatrix.cdk.api.SysLogger; -import com.metamatrix.connector.xml.CachingConnector; +import com.metamatrix.connector.xml.StatefulConnector; import com.metamatrix.connector.xml.XMLConnectorState; /** @@ -84,7 +84,7 @@ super(); } - public Connection getConnection(CachingConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException { + public Connection getConnection(StatefulConnector connector, ExecutionContext context, ConnectorEnvironment environment) throws ConnectorException { return null; } } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -73,7 +73,7 @@ * Optional property - this values specifies how many open file descriptors should be cached * in the storage directory. Increasing this value in heavy load may improve performance * but will use more file descriptors, which are a limited system resource. The default - * is 10. + * is 32. */ public static final String MAX_OPEN_FILES = "metamatrix.buffer.maxOpenFiles"; //$NON-NLS-1$ /** @@ -105,16 +105,8 @@ TupleBuffer createTupleBuffer(List elements, String groupName, TupleSourceType tupleSourceType) throws MetaMatrixComponentException; - /** - * Removes all tuple sources by group name - * @param groupName Tuple source group name - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void removeTupleBuffers(String groupName) - throws MetaMatrixComponentException; - int getMaxProcessingBatches(); + FileStore createFileStore(String name); + } Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/FileStore.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/FileStore.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/FileStore.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -0,0 +1,208 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.ref.PhantomReference; +import java.lang.ref.ReferenceQueue; +import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +import com.metamatrix.api.exception.MetaMatrixComponentException; + +public abstract class FileStore { + + private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>(); + private static final Set<PhantomReference<Object>> REFERENCES = Collections.newSetFromMap(new IdentityHashMap<PhantomReference<Object>, Boolean>()); + + /** + * A customized buffered stream with an exposed buffer + */ + public final class FileStoreOutputStream extends OutputStream { + + private byte[] buffer; + private int count; + private boolean bytesWritten; + + public FileStoreOutputStream(int size) { + this.buffer = new byte[size]; + } + + @Override + public void write(int b) throws IOException { + write(new byte[b], 0, 1); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (count + len <= buffer.length) { + System.arraycopy(b, off, buffer, count, len); + count += len; + return; + } + flushBuffer(); + writeDirect(b, off, len); + } + + private void writeDirect(byte[] b, int off, int len) throws IOException { + try { + FileStore.this.write(b, off, len); + bytesWritten = true; + } catch (MetaMatrixComponentException e) { + throw new IOException(e); + } + } + + private void flushBuffer() throws IOException { + if (count > 0) { + writeDirect(buffer, 0, count); + count = 0; + } + } + + public boolean bytesWritten() { + return bytesWritten; + } + + public byte toByteArray()[] { + return Arrays.copyOf(buffer, count); + } + + @Override + public void close() throws IOException { + if (bytesWritten) { + flushBuffer(); + } + } + + } + + static class CleanupReference extends PhantomReference<Object> { + + private FileStore store; + + public CleanupReference(Object referent, FileStore store) { + super(referent, QUEUE); + this.store = store; + } + + public void cleanup() { + try { + this.store.remove(); + } finally { + this.clear(); + } + } + } + + public void setCleanupReference(Object o) { + REFERENCES.add(new CleanupReference(o, this)); + for (int i = 0; i < 10; i++) { + CleanupReference ref = (CleanupReference)QUEUE.poll(); + if (ref == null) { + break; + } + ref.cleanup(); + REFERENCES.remove(ref); + } + } + + private boolean removed; + + public int read(long fileOffset, byte[] b, int offSet, int length) + throws MetaMatrixComponentException { + if (removed) { + throw new MetaMatrixComponentException("already removed"); //$NON-NLS-1$ + } + return readDirect(fileOffset, b, offSet, length); + } + + protected abstract int readDirect(long fileOffset, byte[] b, int offSet, int length) + throws MetaMatrixComponentException; + + public void readFully(long fileOffset, byte[] b, int offSet, int length) throws MetaMatrixComponentException { + int n = 0; + do { + int count = this.read(fileOffset + n, b, offSet + n, length - n); + if (count < 0) { + throw new MetaMatrixComponentException("not enough bytes available"); //$NON-NLS-1$ + } + n += count; + } while (n < length); + } + + public long write(byte[] bytes) throws MetaMatrixComponentException { + return write(bytes, 0, bytes.length); + } + + public long write(byte[] bytes, int offset, int length) throws MetaMatrixComponentException { + if (removed) { + throw new MetaMatrixComponentException("already removed"); //$NON-NLS-1$ + } + return writeDirect(bytes, offset, length); + } + + protected abstract long writeDirect(byte[] bytes, int offset, int length) throws MetaMatrixComponentException; + + public void remove() { + if (!this.removed) { + this.removed = true; + this.removeDirect(); + } + } + + protected abstract void removeDirect(); + + public InputStream createInputStream() { + return new InputStream() { + private long offset; + + @Override + public int read() throws IOException { + throw new UnsupportedOperationException("buffered reading must be used"); //$NON-NLS-1$ + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + try { + int bytes = FileStore.this.read(offset, b, off, len); + if (bytes != -1) { + this.offset += bytes; + } + return bytes; + } catch (MetaMatrixComponentException e) { + throw new IOException(e); + } + } + }; + } + + public FileStoreOutputStream createOutputStream(int maxMemorySize) { + return new FileStoreOutputStream(maxMemorySize); + } + +} \ No newline at end of file Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/FileStore.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -27,15 +27,15 @@ class ManagedBatch { private int beginRow; - private int endRow; private SoftReference<TupleBatch> batchReference; + private long offset; + private int length; /** * Constructor for ManagedBatch. */ public ManagedBatch(TupleBatch batch) { this.beginRow = batch.getBeginRow(); - this.endRow = batch.getEndRow(); this.batchReference = new SoftReference<TupleBatch>(batch); } @@ -47,16 +47,8 @@ return this.beginRow; } - /** - * Get the end row, inclusive - * @return End row - */ - public int getEndRow() { - return this.endRow; - } - public String toString() { - return "ManagedBatch[" + beginRow + ", " + endRow + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + return "ManagedBatch[" + beginRow + "]"; //$NON-NLS-1$ //$NON-NLS-2$ } public TupleBatch getBatch() { @@ -67,4 +59,20 @@ this.batchReference = new SoftReference<TupleBatch>(batch); } + public int getLength() { + return length; + } + + public long getOffset() { + return offset; + } + + public void setLength(int length) { + this.length = length; + } + + public void setOffset(long offset) { + this.offset = offset; + } + } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,76 +23,13 @@ package com.metamatrix.common.buffer; import java.util.Properties; + import com.metamatrix.api.exception.MetaMatrixComponentException; -/** - * Basic interface for a storage manager. The storage manager deals in - * terms of TupleBatch objects. The basic contract assumed here is that - * batches are requested ({@link #getBatch(TupleSourceID, int)} or removed - * ({@link #removeBatch(TupleSourceID, int)}) <b>only</b> with the same - * beginning row that they were added with. For instance, if batches - * from row 1-100 and row 101-200 were added to the storage manager, it - * would be fine to get batch with begin row 1 and 101 but getting a batch - * with beginning row 50 would fail. This contract was created to minimize - * the amount of batch manipulation required by a storage manager. - */ public interface StorageManager { - /** - * Constant for a StorageManager for database storage - */ - public static final int TYPE_DATABASE = 1; + void initialize(Properties props) throws MetaMatrixComponentException; - /** - * Constant for a StorageManager for file storage - */ - public static final int TYPE_FILE = 2; + FileStore createFileStore(String name); - /** - * Constant for a StorageManager for remote app storage - */ - public static final int TYPE_REMOTE = 3; - - /** - * Initialize the storage manager given the specified properties. Properties - * are typically specific to the particular storage manager. - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void initialize(Properties props) - throws MetaMatrixComponentException; - - /** - * Add a batch to the storage manager. - * @param types a hint to the StorageManager about the types of data in the batch - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types) - throws MetaMatrixComponentException; - - /** - * Returns a batch of tuples, starting at row beginRow. The batch - * will only be found if the batch that was added begins at beginRow. - * @param types a hint to the StorageManager about the types of data in the batch - * @return batch of tuples starting at row beginRow - * @throws TupleSourceNotFoundException indicating the sourceID is unknown - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - TupleBatch getBatch(TupleSourceID sourceID, int beginRow, String[] types) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Remove all batches for the specified tuple source. If no batches exist, - * no exception is thrown. - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void removeBatches(TupleSourceID sourceID); - - /** - * Shut down the Storage Manager. - */ - void shutdown(); } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,8 +22,11 @@ package com.metamatrix.common.buffer; -import java.lang.ref.PhantomReference; -import java.lang.ref.ReferenceQueue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.lang.ref.SoftReference; import java.util.ArrayList; import java.util.Collections; @@ -31,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import com.metamatrix.api.exception.MetaMatrixComponentException; @@ -38,6 +42,7 @@ import com.metamatrix.common.types.Streamable; import com.metamatrix.core.util.Assertion; import com.metamatrix.dqp.DQPPlugin; +import com.metamatrix.query.execution.QueryExecPlugin; public class TupleBuffer { @@ -97,8 +102,7 @@ } // Retrieves the necessary batch based on the currentRow - TupleBatch getBatch() - throws MetaMatrixComponentException{ + TupleBatch getBatch() throws MetaMatrixComponentException{ TupleBatch batch = null; if (currentBatch != null) { batch = currentBatch.get(); @@ -147,38 +151,28 @@ } } - private static class LobReference extends PhantomReference<Streamable<?>> { - - String persistentStreamId; - - public LobReference(Streamable<?> lob) { - super(lob, LOB_QUEUE); - this.persistentStreamId = lob.getPersistenceStreamId(); - } - } - private static final AtomicLong LOB_ID = new AtomicLong(); - private static final ReferenceQueue<Streamable<?>> LOB_QUEUE = new ReferenceQueue<Streamable<?>>(); //construction state private StorageManager manager; - private String groupName; - private TupleSourceID tupleSourceID; + private String tupleSourceID; private List<?> schema; private String[] types; private int batchSize; + + private FileStore store; private int rowCount; private boolean isFinal; private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>(); private ArrayList<List<?>> batchBuffer; + private AtomicInteger referenceCount = new AtomicInteger(1); //lob management private Map<String, Streamable<?>> lobReferences; //references to contained lobs private boolean lobs = true; - private LobReference containingLobReference; //reference to containing lob - public TupleBuffer(StorageManager manager, String groupName, TupleSourceID id, List<?> schema, String[] types, int batchSize) { + public TupleBuffer(StorageManager manager, String id, List<?> schema, String[] types, int batchSize) { this.manager = manager; this.tupleSourceID = id; this.schema = schema; @@ -195,18 +189,12 @@ lobs = false; } } - this.groupName = groupName; } - public String getGroupName() { - return groupName; - } - - public boolean isBatchBufferEmpty() { - return this.batchBuffer == null || this.batchBuffer.isEmpty(); - } - public void addTuple(List<?> tuple) throws MetaMatrixComponentException { + if (lobs) { + correctLobReferences(new List[] {tuple}); + } this.rowCount++; if (batchBuffer == null) { batchBuffer = new ArrayList<List<?>>(batchSize/4); @@ -217,27 +205,62 @@ } } - void saveBatch(boolean finalBatch) throws MetaMatrixComponentException { - if ((batchBuffer == null || batchBuffer.isEmpty()) && !finalBatch) { + public void saveBatch(boolean finalBatch) throws MetaMatrixComponentException { + Assertion.assertTrue(!this.isRemoved()); + if (batchBuffer == null || batchBuffer.isEmpty()) { return; } - List rows = batchBuffer==null?Collections.emptyList():batchBuffer; + List<?> rows = batchBuffer==null?Collections.emptyList():batchBuffer; TupleBatch writeBatch = new TupleBatch(rowCount - rows.size() + 1, rows); if (finalBatch) { writeBatch.setTerminationFlag(true); } - if (writeBatch.getTerminationFlag()) { - this.isFinal = true; + ManagedBatch mbatch = new ManagedBatch(writeBatch); + if (this.store == null) { + this.store = this.manager.createFileStore(this.tupleSourceID); + this.store.setCleanupReference(this); } - correctLobReferences(writeBatch); - ManagedBatch mbatch = new ManagedBatch(writeBatch); + byte[] bytes = convertToBytes(writeBatch); + mbatch.setLength(bytes.length); + mbatch.setOffset(this.store.write(bytes)); this.batches.put(mbatch.getBeginRow(), mbatch); - manager.addBatch(tupleSourceID, writeBatch, types); batchBuffer = null; } + /** + * Convert from an object to a byte array + * @param object Object to convert + * @return Byte array + */ + private byte[] convertToBytes(TupleBatch batch) throws MetaMatrixComponentException { + ObjectOutputStream oos = null; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + oos = new ObjectOutputStream(baos); + + batch.setDataTypes(types); + batch.writeExternal(oos); + oos.flush(); + return baos.toByteArray(); + + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStorageManager.batch_error")); //$NON-NLS-1$ + } finally { + if(oos != null) { + try { + oos.close(); + } catch(IOException e) { + } + } + } + } + public void close() throws MetaMatrixComponentException { - saveBatch(true); + //if there is only a single batch, let it stay in memory + if (!this.batches.isEmpty()) { + saveBatch(true); + } + this.isFinal = true; } public TupleBatch getBatch(int row) throws MetaMatrixComponentException { @@ -251,32 +274,56 @@ if (this.batchBuffer != null && row > rowCount - this.batchBuffer.size()) { return new TupleBatch(rowCount - this.batchBuffer.size() + 1, batchBuffer); } + if (this.batchBuffer != null && !this.batchBuffer.isEmpty()) { + saveBatch(isFinal); + } Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(row); - ManagedBatch batch = null; - if (entry != null && entry.getValue().getEndRow() >= row) { - batch = entry.getValue(); - TupleBatch result = batch.getBatch(); - if (result != null) { - return result; - } - row = batch.getBeginRow(); + Assertion.isNotNull(entry); + ManagedBatch batch = entry.getValue(); + TupleBatch result = batch.getBatch(); + if (result != null) { + return result; + } + try { + byte[] bytes = new byte[batch.getLength()]; + this.store.readFully(batch.getOffset(), bytes, 0, bytes.length); + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais); + + result = new TupleBatch(); + result.setDataTypes(types); + result.readExternal(ois); + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", tupleSourceID)); //$NON-NLS-1$ + } catch (ClassNotFoundException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", tupleSourceID)); //$NON-NLS-1$ } - try { - TupleBatch result = manager.getBatch(tupleSourceID, row, types); - correctLobReferences(result); - if (batch != null) { - batch.setBatchReference(result); - } - return result; - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); + if (lobs) { + correctLobReferences(result.getAllTuples()); } + batch.setBatchReference(result); + return result; } public void remove() { - manager.removeBatches(tupleSourceID); + int count = this.referenceCount.getAndDecrement(); + if (count == 0) { + if (this.store != null) { + this.store.remove(); + this.store = null; + } + if (this.batchBuffer != null) { + this.batchBuffer = null; + } + this.batches.clear(); + } } + public boolean addReference() { + int count = this.referenceCount.addAndGet(1); + return count > 1; + } + public int getRowCount() { return rowCount; } @@ -300,31 +347,7 @@ public void setBatchSize(int batchSize) { this.batchSize = batchSize; } - - public void setContainingLobReference(Streamable<?> s) { - cleanLobTupleSource(); - s.setPersistenceStreamId(tupleSourceID.getStringID()); - Assertion.isNull(this.containingLobReference); - this.containingLobReference = new LobReference(s); - } - - /** - * Attempt to clean up lingering tuplebuffers to lobs that are no longer referenced - */ - private void cleanLobTupleSource() { - for (int i = 0; i < 10; i++) { - String tupleSourceId = null; - LobReference ref = (LobReference)LOB_QUEUE.poll(); - if (ref != null) { - tupleSourceId = ref.persistentStreamId; - } - if (tupleSourceId == null) { - break; - } - this.manager.removeBatches(new TupleSourceID(tupleSourceId)); - } - } - + public Streamable<?> getLobReference(String id) throws MetaMatrixComponentException { Streamable<?> lob = null; if (this.lobReferences != null) { @@ -343,11 +366,7 @@ * @throws MetaMatrixComponentException */ @SuppressWarnings("unchecked") - private void correctLobReferences(TupleBatch batch) throws MetaMatrixComponentException { - if (!lobs) { - return; - } - List[] rows = batch.getAllTuples(); + private void correctLobReferences(List[] rows) throws MetaMatrixComponentException { int columns = schema.size(); // walk through the results and find all the lobs for (int row = 0; row < rows.length; row++) { @@ -387,4 +406,8 @@ return this.tupleSourceID.toString(); } + public boolean isRemoved() { + return this.referenceCount.get() <= 0; + } + } Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,138 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -import java.io.Serializable; - -/** - * Identifier for a tuple source. The tuple source ID contains two - * pieces of information: a unique ID (unique across locations) and - * a location. Both must be represented by a string but different - * systems may use different location descriptions. - */ -public class TupleSourceID implements Serializable { - - private static final String LOCATION_SEPARATOR = ":"; //$NON-NLS-1$ - - private String idValue; - private String location; - - /** - * Creates a tuple source ID given a string version of the ID - the - * location should be encoded within the stringID, in the form - * <location>:<idValue>. - * @param stringID the String ID; may not be null (zero-length is tolerated) - */ - public TupleSourceID(String stringID){ - if(stringID == null){ - throw new IllegalArgumentException("id may not be null"); //$NON-NLS-1$ - } - - int index = stringID.indexOf(LOCATION_SEPARATOR); - if(index >= 0) { - this.location = stringID.substring(0, index); - this.idValue = stringID.substring(index+1); - - } else { - this.location = null; - this.idValue = stringID; - } - } - - /** - * Creates a tuple source ID given a string version of the ID - * and the location. - * @param idValue Unique ID; may not be null (zero-length is tolerated) - * @param location Location string - */ - public TupleSourceID(String idValue, String location){ - if(idValue == null){ - throw new IllegalArgumentException("id may not be null"); //$NON-NLS-1$ - } - - this.idValue = idValue; - this.location = location; - } - - /** - * Get location string - * @return the location; may be null - */ - public String getLocation() { - return this.location; - } - - /** - * Get unique ID value, does not contain location - * @return ID value; never null but possibly zero-length - */ - public String getIDValue() { - return this.idValue; - } - - /** - * Get string combining the unique ID and location in the form - * <location>:<idValue> if a location is specified. Otherwise, - * just the idValue is returned. - */ - public String getStringID() { - if(this.location == null) { - return this.idValue; - } - return this.location + LOCATION_SEPARATOR + this.idValue; - } - - /** - * Compares two TupleSourceIDs for equality. - * @param obj Other object - * @return True if equal - */ - public boolean equals(Object obj) { - if(obj == this) { - return true; - } - - if(! (obj instanceof TupleSourceID)) { - return false; - } - - TupleSourceID other = (TupleSourceID) obj; - return other.getIDValue().equals(getIDValue()); - } - - /** - * Get hash code - * @return hash code - */ - public int hashCode() { - return this.idValue.hashCode(); - } - - /** - * Get combined string representation of TupleSourceID - * @return String representation - */ - public String toString() { - return getStringID(); - } -} Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceNotFoundException.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceNotFoundException.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceNotFoundException.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,75 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -import com.metamatrix.api.exception.MetaMatrixProcessingException; - -/** - * Indicates a TupleSource could not be found. - */ -public class TupleSourceNotFoundException extends MetaMatrixProcessingException { - - /** - * No-arg costructor required by Externalizable semantics - */ - public TupleSourceNotFoundException() { - super(); - } - - /** - * Constructor for TupleSourceNotFoundException. - * @param message - */ - public TupleSourceNotFoundException(String message) { - super(message); - } - - /** - * Constructor for TupleSourceNotFoundException. - * @param code - * @param message - */ - public TupleSourceNotFoundException(String code, String message) { - super(code, message); - } - - /** - * Constructor for TupleSourceNotFoundException. - * @param e - * @param message - */ - public TupleSourceNotFoundException(Throwable e, String message) { - super(e, message); - } - - /** - * Constructor for TupleSourceNotFoundException. - * @param e - * @param code - * @param message - */ - public TupleSourceNotFoundException(Throwable e, String code, String message) { - super(e, code, message); - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,32 +22,32 @@ package com.metamatrix.common.buffer.impl; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.ListIterator; -import java.util.Map; import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import javax.xml.transform.Source; + import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManager; +import com.metamatrix.common.buffer.FileStore; import com.metamatrix.common.buffer.StorageManager; -import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleBuffer; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.log.LogManager; import com.metamatrix.common.types.DataTypeManager; +import com.metamatrix.common.types.InputStreamFactory; +import com.metamatrix.common.types.SQLXMLImpl; +import com.metamatrix.common.types.SourceTransform; +import com.metamatrix.common.types.StandardXMLTranslator; +import com.metamatrix.common.types.Streamable; +import com.metamatrix.common.types.XMLType; import com.metamatrix.common.util.PropertiesUtils; +import com.metamatrix.core.MetaMatrixRuntimeException; import com.metamatrix.core.log.MessageLevel; import com.metamatrix.core.util.Assertion; import com.metamatrix.dqp.util.LogConstants; -import com.metamatrix.query.execution.QueryExecPlugin; +import com.metamatrix.query.processor.xml.XMLUtil; import com.metamatrix.query.sql.symbol.Expression; /** @@ -56,16 +56,11 @@ */ public class BufferManagerImpl implements BufferManager, StorageManager { - private String lookup; - // Configuration private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE; private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE; private int maxProcessingBatches = BufferManager.DEFAULT_MAX_PROCESSING_BATCHES; - private Map<TupleSourceID, TupleBuffer> tupleSourceMap = new ConcurrentHashMap<TupleSourceID, TupleBuffer>(); - private Map<String, Set<TupleSourceID>> groupInfos = new HashMap<String, Set<TupleSourceID>>(); - private StorageManager diskMgr; private AtomicLong currentTuple = new AtomicLong(0); @@ -120,17 +115,43 @@ public TupleBuffer createTupleBuffer(List elements, String groupName, TupleSourceType tupleSourceType) throws MetaMatrixComponentException { - TupleSourceID newID = new TupleSourceID(String.valueOf(this.currentTuple.getAndIncrement()), this.lookup); - Set<TupleSourceID> tupleGroupInfo = getGroupInfo(groupName, true); - TupleBuffer tupleBuffer = new TupleBuffer(this, groupName, newID, elements, getTypeNames(elements), getProcessorBatchSize()); - tupleGroupInfo.add(newID); - tupleSourceMap.put(newID, tupleBuffer); + String newID = String.valueOf(this.currentTuple.getAndIncrement()); + TupleBuffer tupleBuffer = new TupleBuffer(this, newID, elements, getTypeNames(elements), getProcessorBatchSize()); if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Creating TupleBuffer:", newID, "of type "+tupleSourceType}); //$NON-NLS-1$ //$NON-NLS-2$ } return tupleBuffer; } + @Override + public FileStore createFileStore(String name) { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating FileStore:", name); //$NON-NLS-1$ + } + return this.diskMgr.createFileStore(name); + } + + @Override + public void initialize(Properties props) throws MetaMatrixComponentException { + PropertiesUtils.setBeanProperties(this, props, "metamatrix.buffer"); //$NON-NLS-1$ + DataTypeManager.addSourceTransform(Source.class, new SourceTransform<Source, XMLType>() { + @Override + public XMLType transform(Source value) { + if (value instanceof InputStreamFactory) { + return new XMLType(new SQLXMLImpl((InputStreamFactory)value)); + } + StandardXMLTranslator sxt = new StandardXMLTranslator(value, null); + SQLXMLImpl sqlxml; + try { + sqlxml = XMLUtil.saveToBufferManager(BufferManagerImpl.this, sxt, Streamable.STREAMING_BATCH_SIZE_IN_BYTES); + } catch (MetaMatrixComponentException e) { + throw new MetaMatrixRuntimeException(e); + } + return new XMLType(sqlxml); + } + }); + } + /** * Gets the data type names for each of the input expressions, in order. * @param expressions List of Expressions @@ -149,119 +170,7 @@ return types; } - /** - * Remove all the tuple sources with the specified group name. Typically the group - * name is really a session identifier and this is called to remove all tuple sources - * used by a particular session. - * @param groupName Name of the group - * @throws MetaMatrixComponentException If internal server error occurs - */ - public void removeTupleBuffers(String groupName) - throws MetaMatrixComponentException { - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleBuffers for group", groupName}); //$NON-NLS-1$ - } - - Set<TupleSourceID> tupleGroupInfo = null; - synchronized(groupInfos) { - tupleGroupInfo = groupInfos.remove(groupName); - } - if (tupleGroupInfo == null) { - return; - } - List<TupleSourceID> tupleSourceIDs = null; - synchronized (tupleGroupInfo) { - tupleSourceIDs = new ArrayList<TupleSourceID>(tupleGroupInfo); - } - for (TupleSourceID tsID : tupleSourceIDs) { - try { - TupleBuffer tupleBuffer = getTupleBuffer(tsID); - tupleBuffer.remove(); - } catch(TupleSourceNotFoundException e) { - // ignore and go on - } - } - } - - /** - * Get the tuple source info for the specified tuple source ID - * @param tupleSourceID Tuple source to get - * @return Tuple source info - * @throws TupleSourceNotFoundException If not found - */ - public TupleBuffer getTupleBuffer(TupleSourceID tupleSourceID) throws TupleSourceNotFoundException { - TupleBuffer info = this.tupleSourceMap.get(tupleSourceID); - - if(info == null) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$ - } - return info; - } - - /** - * Gets a TupleGroupInfo representing the group - * @param groupName Logical grouping name for tuple sources. Can be null. - * @return TupleGroupInfo for the group. Never null. - * @since 4.3 - */ - private Set<TupleSourceID> getGroupInfo(String groupName, boolean create) { - Set<TupleSourceID> info = null; - synchronized(groupInfos) { - info = groupInfos.get(groupName); - if (info == null && create) { - // If it doesn't exist, create a new one - info = Collections.synchronizedSet(new HashSet<TupleSourceID>()); - groupInfos.put(groupName, info); - } - } - return info; - } - - @Override - public void addBatch(TupleSourceID sourceID, TupleBatch batch, - String[] types) throws MetaMatrixComponentException { - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"AddTupleBatch for", sourceID, "with " + batch.getRowCount() + " rows"}); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - this.diskMgr.addBatch(sourceID, batch, types); - } - - @Override - public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, - String[] types) throws TupleSourceNotFoundException, - MetaMatrixComponentException { - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Getting tupleBatch for", sourceID, "beginRow:", beginRow}); //$NON-NLS-1$ //$NON-NLS-2$ - } - return this.diskMgr.getBatch(sourceID, beginRow, types); - } - - @Override - public void initialize(Properties props) - throws MetaMatrixComponentException { - this.lookup = "local"; //$NON-NLS-1$ - PropertiesUtils.setBeanProperties(this, props, "metamatrix.buffer"); //$NON-NLS-1$ - } - - @Override - public void removeBatches(TupleSourceID sourceID) { - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleBuffer:", sourceID}); //$NON-NLS-1$ - } - TupleBuffer tupleBuffer = this.tupleSourceMap.remove(sourceID); - if (tupleBuffer != null) { - Set<TupleSourceID> ids = getGroupInfo(tupleBuffer.getGroupName(), false); - if (ids != null) { - ids.remove(sourceID); - } - } - this.diskMgr.removeBatches(sourceID); - } - - @Override public void shutdown() { - this.diskMgr.shutdown(); } } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,56 +22,145 @@ package com.metamatrix.common.buffer.impl; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.RandomAccessFile; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.TreeMap; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManager; +import com.metamatrix.common.buffer.FileStore; import com.metamatrix.common.buffer.StorageManager; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.log.LogManager; import com.metamatrix.common.util.PropertiesUtils; import com.metamatrix.core.log.MessageLevel; import com.metamatrix.core.util.Assertion; import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.query.execution.QueryExecPlugin; + /** - * This class stores batches in files on disk in a specified directory. Every - * tuple source gets it's own set of files, named b_<i>id</i>_<i>num</i> where <i>id</i> is the - * unique id value of the TupleSourceID, and <i>num</i> is a file counter for the files - * associated with the TupleSourceID. Batches are stored random access into the - * file, typically but not necessarily in order. An in memory data structure stores - * info about all open files and where the batches are in the file (file pointers and - * lengths). + * Implements file storage that automatically splits large files and limits the number of open files. */ public class FileStorageManager implements StorageManager { - + private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$ + + private class FileInfo { + private File file; + private RandomAccessFile fileData; // may be null if not open + public FileInfo(File file) { + this.file = file; + } + + public RandomAccessFile open() throws FileNotFoundException { + if(this.fileData == null) { + this.fileData = fileCache.remove(this.file); + if (this.fileData == null) { + this.fileData = new RandomAccessFile(file, "rw"); //$NON-NLS-1$ + } + } + return this.fileData; + } + + public void close() { + fileCache.put(this.file, this.fileData); + this.fileData = null; + } + + public void delete() { + RandomAccessFile raf = fileCache.remove(this.file); + if (raf != null) { + try { + raf.close(); + } catch (IOException e) { + } + } + file.delete(); + } + + public String toString() { + return "FileInfo<" + file.getName() + ", has fileData = " + (fileData != null) + ">"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + } + + public class DiskStore extends FileStore { + private String name; + private TreeMap<Long, FileInfo> storageFiles = new TreeMap<Long, FileInfo>(); + + public DiskStore(String name) { + this.name = name; + } + + public synchronized int readDirect(long fileOffset, byte[] b, int offSet, int length) throws MetaMatrixComponentException { + Map.Entry<Long, FileInfo> entry = storageFiles.floorEntry(fileOffset); + Assertion.isNotNull(entry); + FileInfo fileInfo = entry.getValue(); + try { + RandomAccessFile fileAccess = fileInfo.open(); + fileAccess.seek(fileOffset - entry.getKey()); + return fileAccess.read(b, offSet, length); + } catch (IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ + } finally { + fileInfo.close(); + } + } + + public synchronized long writeDirect(byte[] bytes, int offset, int length) throws MetaMatrixComponentException { + Map.Entry<Long, FileInfo> entry = this.storageFiles.lastEntry(); + boolean createNew = false; + FileInfo fileInfo = null; + long fileOffset = 0; + if (entry == null) { + createNew = true; + } else { + fileInfo = entry.getValue(); + fileOffset = entry.getKey(); + createNew = entry.getValue().file.length() + length > getMaxFileSize(); + } + if (createNew) { + FileInfo newFileInfo = new FileInfo(createFile(name, storageFiles.size())); + if (fileInfo != null) { + fileOffset += fileInfo.file.length(); + } + storageFiles.put(fileOffset, newFileInfo); + fileInfo = newFileInfo; + } + try { + RandomAccessFile fileAccess = fileInfo.open(); + long pointer = fileAccess.length(); + fileAccess.setLength(pointer + length); + fileAccess.seek(pointer); + fileAccess.write(bytes, offset, length); + return fileOffset + pointer; + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ + } finally { + fileInfo.close(); + } + } + + public synchronized void removeDirect() { + for (FileInfo info : storageFiles.values()) { + info.delete(); + } + } + + } + // Initialization - private int maxOpenFiles = 10; + private int maxOpenFiles = 32; private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB private String directory; private File dirFile; // State - private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new HashMap<TupleSourceID, TupleSourceInfo>(); private Map<File, RandomAccessFile> fileCache = Collections.synchronizedMap(new LinkedHashMap<File, RandomAccessFile>() { @Override protected boolean removeEldestEntry( @@ -86,7 +175,7 @@ return false; } }); - + /** * Initialize with properties * @param props Initialization properties @@ -111,319 +200,42 @@ } // Set up max number of open file descriptors - maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManager.MAX_OPEN_FILES, 10); + maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManager.MAX_OPEN_FILES, 32); // Set the max file size maxFileSize = PropertiesUtils.getIntProperty(props, BufferManager.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB } - public String getDirectory() { - return directory; - } - - /** - * Look up tuple source info and possibly create. First the file map is used to find an - * existing file info. If the info is found it is returned. If not, then - * a TupleSourceInfo is created according to shouldCreate flag - * @param sourceID Source identifier - * @param shouldCreate true if this method should create info related to the tuple source ID if it does not exist. - * @return All the tuple source info or null if shouldCreate == false and no tuple source info was found - */ - private TupleSourceInfo getTupleSourceInfo(TupleSourceID sourceID, boolean shouldCreate) { - - // Try to find in cache - synchronized(tupleSourceMap) { - TupleSourceInfo info = tupleSourceMap.get(sourceID); - if(info == null && shouldCreate) { - info = new TupleSourceInfo(); - tupleSourceMap.put(sourceID, info); - } - return info; - } - } - - /** - * Creates a new file to the specified TupleSourceID - * @param sourceID The TupleSourceID - * @param fileNumber a number uniquely identifying this file within the set of tuple source files. - * @return the newly created file with the name b_<i>sourceID</i>_<i>fileNumber</i> - * @throws MetaMatrixComponentException if a file with this name already exists in the directory, or if it cannot be created - * @since 4.2 - */ - private File createFile(TupleSourceID sourceID, int fileNumber) throws MetaMatrixComponentException { - File storageFile = new File(this.directory, FILE_PREFIX + sourceID.getIDValue() + "_" + fileNumber); //$NON-NLS-1$ - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Creating temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$ - } + File createFile(String name, int fileNumber) throws MetaMatrixComponentException { try { - boolean created = storageFile.createNewFile(); - if(!created) { - throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.file_exists", storageFile.getAbsoluteFile())); //$NON-NLS-1$ - } - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", storageFile.getAbsoluteFile())); //$NON-NLS-1$ - } - return storageFile; - } - - /** - * Add a batch to the storage manager. Because we don't implement remove, it's - * possible that this batch already exists, in which case nothing is done. - * @param sourceID Source identifier - * @param batch Batch to add - */ - public void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types) - throws MetaMatrixComponentException { - - // Defect 13342 - addBatch method now creates spill files if the total bytes exceeds the max file size limit - TupleSourceInfo tsInfo = getTupleSourceInfo(sourceID, true); - synchronized (tsInfo) { - if (tsInfo.isRemoved) { - return; - } - Integer batchKey = new Integer(batch.getBeginRow()); - if (tsInfo.tupleBatchPointers.containsKey(batchKey)) { - return; - } - byte[] bytes = convertToBytes(batch, types); - if (bytes.length > maxFileSize) { - LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, "Detected an attempt to save a batch (" + sourceID + ", begin=" + batch.getBeginRow()+ ", size=" + bytes.length + ") larger than the buffer max file size setting of " + maxFileSize + " bytes. The buffer manager will ignore the max file size setting for this batch, and create a buffer file dedicated to this batch. It may be necessary to reduce the buffer batch setting or increase the buffer max file size setting."); //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ - } - FileInfo fileInfo = tsInfo.getMostRecentlyCreatedFile(); - if (fileInfo == null || - (/*fileInfo.file.length() != 0 && */fileInfo.file.length() + bytes.length > maxFileSize)) { - // Create and add - fileInfo = new FileInfo(createFile(sourceID, tsInfo.storageFiles.size())); - tsInfo.storageFiles.add(fileInfo); - } - long pointer = 0; - + File storageFile = File.createTempFile(FILE_PREFIX + name + "_" + String.valueOf(fileNumber) + "_", null, this.dirFile); //$NON-NLS-1$ //$NON-NLS-2$ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Adding batch to storage area file " + fileInfo.file.getAbsoluteFile() + " [ sourceID: " + sourceID + "batch: " + batch + " ]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ + LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Created temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$ } - try { - // Get access to the file and remember whether we had to open it or not - fileInfo.open(); - RandomAccessFile fileAccess = fileInfo.getAccess(); - - // Store the batch in the file - pointer = fileAccess.length(); - fileAccess.setLength(pointer + bytes.length); - fileAccess.seek(pointer); - fileAccess.write(bytes); - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } finally { - fileInfo.close(); - } - - // Update the pointers - tsInfo.tupleBatchPointers.put(batchKey, new PointerInfo(fileInfo, pointer, bytes.length)); - } - } - - /** - * Convert from an object to a byte array - * @param object Object to convert - * @return Byte array - */ - private byte[] convertToBytes(TupleBatch batch, String[] types) throws MetaMatrixComponentException { - ObjectOutputStream oos = null; - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - oos = new ObjectOutputStream(baos); - - batch.setDataTypes(types); - batch.writeExternal(oos); - oos.flush(); - return baos.toByteArray(); - + return storageFile; } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStorageManager.batch_error")); //$NON-NLS-1$ - } finally { - if(oos != null) { - try { - oos.close(); - } catch(IOException e) { - } - } + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", name + "_" + fileNumber)); //$NON-NLS-1$ //$NON-NLS-2$ } } - - /** - * Get a batch from the storage manager based on the beginRow. - * @param sourceID Source identifier - * @param beginRow Beginning row of batch to retrieve - * @return Batch retrieved - */ - public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, String[] types) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(sourceID, false); - if(info == null) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ - } - - byte[] bytes = null; - synchronized(info) { - if(info.isRemoved) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ - } - // Find pointer - PointerInfo pointerInfo = info.tupleBatchPointers.get(new Integer(beginRow)); - Assertion.isNotNull(pointerInfo); - - FileInfo fileInfo = pointerInfo.fileInfo; - // Get access to the file - RandomAccessFile fileAccess = null; - try { - fileInfo.open(); - fileAccess = fileInfo.getAccess(); - fileAccess.seek(pointerInfo.pointer); - bytes = new byte[pointerInfo.length]; - fileAccess.readFully(bytes); - ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - ObjectInputStream ois = new ObjectInputStream(bais); - - TupleBatch batch = new TupleBatch(); - batch.setDataTypes(types); - batch.readExternal(ois); - return batch; - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } finally { - fileInfo.close(); - } - } + + public FileStore createFileStore(String name) { + return new DiskStore(name); } - - /** - * Remove all batches for a sourceID. Before removal, the file is closed. - * @param sourceID Tuple source ID - */ - public void removeBatches(TupleSourceID sourceID) { - TupleSourceInfo info = null; - // Remove info from the file map - synchronized(tupleSourceMap) { - info = tupleSourceMap.remove(sourceID); - } - - // Didn't find a file - if(info == null) { - return; - } - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing storage for " + sourceID); //$NON-NLS-1$ - } - - synchronized(info) { - // If open, close the file and decrement the open file counter - for (int i = 0; i < info.storageFiles.size(); i++) { - FileInfo fileInfo = info.storageFiles.get(i); - fileInfo.delete(); - } - // Delete the file and mark info as being removed - info.isRemoved = true; - } - } - - /** - * This method removes all storage area files by walking through the file info - * map and closing and removing each file. - */ - public synchronized void shutdown() { - - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing all storage area files "); //$NON-NLS-1$ - - List<TupleSourceID> ids = null; - synchronized (tupleSourceMap) { - ids = new LinkedList<TupleSourceID>(tupleSourceMap.keySet()); - } - - for (TupleSourceID key : ids) { - removeBatches(key); - } - - tupleSourceMap = null; - } - + + public long getMaxFileSize() { + return maxFileSize; + } + + public String getDirectory() { + return directory; + } + + Map<File, RandomAccessFile> getFileCache() { + return fileCache; + } + public int getOpenFiles() { - return this.fileCache.size(); + return this.fileCache.size(); } - private class FileInfo { - private File file; - private RandomAccessFile fileData; // may be null if not open - - public FileInfo(File file) { - this.file = file; - } - - public void open() throws FileNotFoundException { - if(this.fileData == null) { - this.fileData = fileCache.remove(this.file); - if (this.fileData == null) { - this.fileData = new RandomAccessFile(file, "rw"); //$NON-NLS-1$ - } - } - } - - public RandomAccessFile getAccess() { - return this.fileData; - } - - public void close() { - if(this.fileData != null) { - fileCache.put(file, this.fileData); - this.fileData = null; - } - } - - public void delete() { - if (this.fileData == null) { - this.fileData = fileCache.remove(this.file); - } - if (this.fileData != null) { - try { - this.fileData.close(); - } catch(Exception e) { - } - this.fileData = null; - } - file.delete(); - } - - public String toString() { - return "FileInfo<" + file.getName() + ", has fileData = " + (fileData != null) + ">"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - } - - private static class PointerInfo { - FileInfo fileInfo; - public long pointer; - public int length; - - public PointerInfo(FileInfo fileInfo, long pointer, int length) { - this.fileInfo = fileInfo; - this.pointer = pointer; - this.length = length; - } - } - - private static class TupleSourceInfo { - Map<Integer, PointerInfo> tupleBatchPointers = new HashMap<Integer, PointerInfo>(); - List<FileInfo> storageFiles = new ArrayList<FileInfo>(2); // Stores all the FileInfos for this tupleSource - private boolean isRemoved = false; - - FileInfo getMostRecentlyCreatedFile() { - if (storageFiles.isEmpty()) { - return null; - } - return storageFiles.get(storageFiles.size() - 1); - } - } - } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,83 +22,58 @@ package com.metamatrix.common.buffer.impl; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.nio.ByteBuffer; import java.util.Properties; import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.common.buffer.FileStore; import com.metamatrix.common.buffer.StorageManager; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.query.execution.QueryExecPlugin; public class MemoryStorageManager implements StorageManager { - // TupleSourceID -> List<TupleBatch> (ordered by startRow) - private Map<TupleSourceID, Map<Integer, TupleBatch>> storage = Collections.synchronizedMap(new HashMap<TupleSourceID, Map<Integer, TupleBatch>>()); - /** * @see StorageManager#initialize(Properties) */ public void initialize(Properties props) throws MetaMatrixComponentException { } - /** - * @see StorageManager#addBatch(TupleSourceID, TupleBatch) - */ - public void addBatch(TupleSourceID storageID, TupleBatch batch, String[] types) - throws MetaMatrixComponentException { - - Map<Integer, TupleBatch> batches = null; - synchronized(this.storage) { - batches = storage.get(storageID); - if(batches == null) { - batches = new HashMap<Integer, TupleBatch>(); - this.storage.put(storageID, batches); - } - } - - synchronized(batches) { - batches.put(batch.getBeginRow(), batch); - } - } - - /** - * @see StorageManager#getBatch(TupleSourceID, int, int) - */ - public TupleBatch getBatch(TupleSourceID storageID, int beginRow, String[] types) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - Map<Integer, TupleBatch> batches = storage.get(storageID); - - if(batches == null) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", storageID)); //$NON-NLS-1$ - } - - synchronized(batches) { - TupleBatch batch = batches.get(beginRow); - if(batch == null) { - throw new MetaMatrixComponentException("unknown batch"); //$NON-NLS-1$ - } - return batch; - } - } - - /** - * @see StorageManager#removeStorageArea(TupleSourceID) - */ - public void removeBatches(TupleSourceID storageID) { - storage.remove(storageID); - } - - /** - * @see StorageManager#shutdown() - */ - public void shutdown() { - this.storage.clear(); - this.storage = null; - } - + @Override + public FileStore createFileStore(String name) { + return new FileStore() { + private ByteBuffer buffer = ByteBuffer.allocate(2 << 15); + private int end; + + @Override + public synchronized long writeDirect(byte[] bytes, int offset, int length) throws MetaMatrixComponentException { + if (end + length > buffer.capacity()) { + ByteBuffer newBuffer = ByteBuffer.allocate(buffer.capacity() * 2 + length); + newBuffer.put(buffer); + buffer = newBuffer; + } + buffer.position(end); + buffer.put(bytes, offset, length); + long result = end; + end += length; + return result; + } + + @Override + public synchronized void removeDirect() { + buffer = ByteBuffer.allocate(0); + } + + @Override + public synchronized int readDirect(long fileOffset, byte[] b, int offset, int length) + throws MetaMatrixComponentException { + if (fileOffset >= end) { + return -1; + } + int position = (int)fileOffset; + buffer.position(position); + length = Math.min(length, end - position); + buffer.get(b, offset, length); + return length; + } + }; + } } \ No newline at end of file Modified: trunk/engine/src/main/java/com/metamatrix/query/function/FunctionTree.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/function/FunctionTree.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/function/FunctionTree.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -70,7 +70,9 @@ /** * Function selection use: Map of function name (uppercase) to List of FunctionMethod */ - private Map functionsByName = new HashMap(); + private Map<String, List<FunctionMethod>> functionsByName = new HashMap<String, List<FunctionMethod>>(); + + private Set<FunctionMethod> allFunctions = new HashSet<FunctionMethod>(); /** * Function lookup and invocation use: Function name (uppercase) to Map (recursive tree) @@ -160,11 +162,7 @@ * newer indistinguishable functions will just not be added. */ private boolean containsIndistinguishableFunction(FunctionMethod method){ - List knownMethods = (List) functionsByName.get(method.getName().toUpperCase()); - if(knownMethods == null) { - return false; - } - return knownMethods.contains(method); + return allFunctions.contains(method); } /** @@ -202,12 +200,13 @@ methods.add(method); // Add method to list by function name - List knownMethods = (List) functionsByName.get(nameKey); + List<FunctionMethod> knownMethods = functionsByName.get(nameKey); if(knownMethods == null) { - knownMethods = new ArrayList(); + knownMethods = new ArrayList<FunctionMethod>(); functionsByName.put(nameKey, knownMethods); } knownMethods.add(method); + allFunctions.add(method); } /** @@ -265,7 +264,7 @@ */ List<FunctionMethod> findFunctionMethods(String name, int args) { final List<FunctionMethod> allMatches = new ArrayList<FunctionMethod>(); - List<FunctionMethod> methods = (List<FunctionMethod>) functionsByName.get(name.toUpperCase()); + List<FunctionMethod> methods = functionsByName.get(name.toUpperCase()); if(methods == null || methods.size() == 0) { return allMatches; } @@ -305,8 +304,6 @@ inputTypes.set(inputTypes.size() - 1, Array.newInstance(inputTypes.get(inputTypes.size() - 1), 0).getClass()); } - inputTypes.add(0, CommandContext.class); - // Get return type FunctionParameter outputParam = method.getOutputParameter(); Class outputType = null; @@ -323,10 +320,10 @@ ReflectionHelper helper = new ReflectionHelper(methodClass); try { invocationMethod = helper.findBestMethodWithSignature(method.getInvocationMethod(), inputTypes); - requiresContext = true; } catch (NoSuchMethodException e) { - inputTypes = inputTypes.subList(1, inputTypes.size()); + inputTypes.add(0, CommandContext.class); invocationMethod = helper.findBestMethodWithSignature(method.getInvocationMethod(), inputTypes); + requiresContext = true; } } catch (ClassNotFoundException e) { // Failed to load class, so can't load method - this will fail at invocation time. @@ -339,6 +336,8 @@ if(invocationMethod != null && !FunctionTree.isValidMethod(invocationMethod)) { throw new MetaMatrixRuntimeException(ErrorMessageKeys.FUNCTION_0047, QueryPlugin.Util.getString(ErrorMessageKeys.FUNCTION_0047, new Object[]{method.getInvocationClass(), invocationMethod, inputTypes})); } + } else { + inputTypes.add(0, CommandContext.class); } FunctionDescriptor descriptor = new FunctionDescriptor(method.getName(), method.getPushdown(), types, outputType, invocationMethod, requiresContext, method.isNullDependent(), method.getDeterministic()); Modified: trunk/engine/src/main/java/com/metamatrix/query/function/metadata/FunctionMethod.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/function/metadata/FunctionMethod.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/function/metadata/FunctionMethod.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,6 +23,8 @@ package com.metamatrix.query.function.metadata; import java.io.Serializable; +import java.util.Arrays; + import com.metamatrix.core.util.HashCodeUtil; /** @@ -333,12 +335,7 @@ public int hashCode() { int hash = HashCodeUtil.hashCode(0, name); if(inputParameters != null) { - hash = HashCodeUtil.hashCode(hash, inputParameters.length); - - // Base hash only on first input parameter type, not all, for performance - if(inputParameters.length > 0 && inputParameters[0] != null) { - hash = HashCodeUtil.hashCode(hash, inputParameters[0].getType()); - } + hash = HashCodeUtil.hashCode(hash, Arrays.hashCode(inputParameters)); } return hash; } @@ -353,9 +350,8 @@ public boolean equals(Object obj) { if(obj == this) { return true; - } else if(obj == null) { - return false; - } else if(obj instanceof FunctionMethod) { + } + if(obj instanceof FunctionMethod) { FunctionMethod other = (FunctionMethod) obj; // Compare # of parameters - do this first as it is much faster than name compare @@ -390,10 +386,8 @@ // Found no discrepancies, must be equal return true; - } else { - // Can't compare object of different type - return false; - } + } + return false; } /** Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureCriteriaProcessor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureCriteriaProcessor.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureCriteriaProcessor.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -29,7 +29,6 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.ExpressionEvaluationException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.core.util.Assertion; import com.metamatrix.query.rewriter.QueryRewriter; import com.metamatrix.query.sql.lang.CompareCriteria; Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -28,7 +28,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.sql.lang.Criteria; import com.metamatrix.query.util.CommandContext; Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -176,15 +176,16 @@ break; } - TupleBuffer activeID = createTupleBuffer(); - activeTupleBuffers.add(activeID); + TupleBuffer sublist = createTupleBuffer(); + activeTupleBuffers.add(sublist); if (this.mode == Mode.SORT) { //perform a stable sort Collections.sort(workingTuples, comparator); } for (List<?> list : workingTuples) { - activeID.addTuple(list); + sublist.addTuple(list); } + sublist.saveBatch(false); } if (this.activeTupleBuffers.isEmpty()) { @@ -241,7 +242,7 @@ id.remove(); } } - + merged.saveBatch(false); this.activeTupleBuffers.add(merged); masterSortIndex = masterSortIndex - sortedIndex + 1; if (masterSortIndex < 0) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -25,9 +25,10 @@ import java.io.IOException; import java.io.Reader; import java.io.StringReader; -import java.io.StringWriter; +import java.io.Writer; import java.nio.charset.Charset; import java.sql.SQLException; +import java.sql.SQLXML; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -63,13 +64,14 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; +import com.metamatrix.common.buffer.FileStore; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.log.LogManager; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.Streamable; +import com.metamatrix.common.types.XMLTranslator; import com.metamatrix.common.types.XMLType; import com.metamatrix.core.util.Assertion; import com.metamatrix.query.execution.QueryExecPlugin; @@ -106,7 +108,7 @@ // is document in progress currently? boolean docInProgress = false; - private TupleBuffer docInProgressTupleSourceId; + private FileStore docInProgressStore; // Post-processing private String styleSheet; @@ -226,31 +228,26 @@ // if the chunk size is less than one chunk unit then send this as string based xml. if (!this.docInProgress && chunk.isLast()) { - xml = new XMLType(new SQLXMLImpl(new String(chunk.getBytes(), Charset.forName("UTF-16")), getProperties())); //$NON-NLS-1$ + xml = new XMLType(new SQLXMLImpl(chunk.getBytes())); } else { // if this is the first chunk, then create a tuple source id for this sequence of chunks if (!this.docInProgress) { this.docInProgress = true; - this.docInProgressTupleSourceId = XMLUtil.createXMLTupleSource(this.bufferMgr, this.getContext().getConnectionID()); + this.docInProgressStore = this.bufferMgr.createFileStore("xml"); //$NON-NLS-1$ } // now save the chunk of data to the buffer manager and move on. - this.docInProgressTupleSourceId.addTuple(Arrays.asList(chunk)); - + this.docInProgressStore.write(chunk.getBytes()); // now document is finished, so create a xml object and return to the client. if (chunk.isLast()) { - this.docInProgressTupleSourceId.close(); - // we want this to be naturally feed by chunks whether inside // or out side the processor - xml = new XMLType(XMLUtil.getFromBufferManager(this.docInProgressTupleSourceId, getProperties())); - this.docInProgressTupleSourceId.setContainingLobReference(xml); - + xml = new XMLType(XMLUtil.createSQLXML(this.docInProgressStore)); //reset current document state. this.docInProgress = false; - this.docInProgressTupleSourceId = null; + this.docInProgressStore = null; } else { return null; //need to continue processing } @@ -289,7 +286,7 @@ if (doc.isFinished()) { this.env.setDocumentInProgress(null); } - byte[] bytes = Arrays.copyOfRange(new String(chunk).getBytes(Charset.forName("UTF-16")), 2, chunk.length * 2 + 2); //$NON-NLS-1$ + byte[] bytes = new String(chunk).getBytes(Charset.forName(Streamable.ENCODING)); return new LobChunk(bytes, doc.isFinished()); } } @@ -501,37 +498,40 @@ // get a reader object for the style sheet Reader styleReader = new StringReader(styleSheet); // construct a Xlan source object for the syle sheet - Source styleSource = new StreamSource(styleReader); + final Source styleSource = new StreamSource(styleReader); // get a reader object for the xml results //Reader xmlReader = new StringReader(xmlResults); // construct a Xlan source object for the xml results - Source xmlSource = new StreamSource(xmlResults.getCharacterStream()); + final Source xmlSource = new StreamSource(xmlResults.getCharacterStream()); // Convert the output target for use in Xalan-J 2 - StringWriter resultOut = new StringWriter(); - StreamResult result = new StreamResult(resultOut); + SQLXML result = XMLUtil.saveToBufferManager(bufferMgr, new XMLTranslator() { + + @Override + public void translate(Writer writer) throws IOException { + try { + // get the Xalan-J 2 XSLT transformer + TransformerFactory factory = new TransformerFactoryImpl(); + Transformer transformer = factory.newTransformer(styleSource); - try { - // get the Xalan-J 2 XSLT transformer - TransformerFactory factory = new TransformerFactoryImpl(); - Transformer transformer = factory.newTransformer(styleSource); + /* + * To use this line, the system property "javax.xml.transform.TransformerFactory" + * needs to be set to "com.icl.saxon.TransformerFactoryImpl" or desired + * TransformerFactory classname. See com.metamatrix.jdbc.TestXMLQuery + */ + //Transformer transformer = TransformerFactory.newInstance().newTransformer(styleSource); - /* - * To use this line, the system property "javax.xml.transform.TransformerFactory" - * needs to be set to "com.icl.saxon.TransformerFactoryImpl" or desired - * TransformerFactory classname. See com.metamatrix.jdbc.TestXMLQuery - */ - //Transformer transformer = TransformerFactory.newInstance().newTransformer(styleSource); + // Feed the resultant I/O stream into the XSLT processor + transformer.transform(xmlSource, new StreamResult(writer)); + } catch(Exception e) { + throw new IOException(QueryExecPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0046), e); + } + } + }, chunkSize); - // Feed the resultant I/O stream into the XSLT processor - transformer.transform(xmlSource, result); - } catch(Exception e) { - throw new MetaMatrixComponentException(e, ErrorMessageKeys.PROCESSOR_0046, QueryExecPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0046)); - } - // obtain the stringified XML results for the - xmlResults = new XMLType(new SQLXMLImpl(resultOut.toString(), props)); + xmlResults = new XMLType(result); } return xmlResults; Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,141 +22,69 @@ package com.metamatrix.query.processor.xml; +import java.io.BufferedInputStream; import java.io.IOException; -import java.io.Reader; -import java.sql.SQLException; -import java.sql.SQLXML; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.TupleBuffer; -import com.metamatrix.common.buffer.BufferManager.TupleSourceType; -import com.metamatrix.common.lob.ByteLobChunkStream; -import com.metamatrix.common.lob.LobChunk; -import com.metamatrix.common.lob.LobChunkInputStream; -import com.metamatrix.common.lob.LobChunkProducer; -import com.metamatrix.common.log.LogManager; -import com.metamatrix.common.types.DataTypeManager; +import com.metamatrix.common.buffer.FileStore; +import com.metamatrix.common.buffer.FileStore.FileStoreOutputStream; +import com.metamatrix.common.types.InputStreamFactory; import com.metamatrix.common.types.SQLXMLImpl; -import com.metamatrix.common.types.XMLReaderFactory; -import com.metamatrix.dqp.DQPPlugin; -import com.metamatrix.dqp.util.LogConstants; -import com.metamatrix.query.sql.symbol.ElementSymbol; +import com.metamatrix.common.types.Streamable; +import com.metamatrix.common.types.XMLTranslator; /** * Utility methods to be used with the XML and XQuery processing. */ public class XMLUtil { - + /** * This method saves the given XML object to the buffer manager's disk process - * and returns the id which is saved under. + * Documents less than the maxMemorySize will be held directly in memory */ - public static TupleBuffer saveToBufferManager(BufferManager bufferMgr, String tupleGroupName, SQLXML srcXML, int chunkSize) + public static SQLXMLImpl saveToBufferManager(BufferManager bufferMgr, XMLTranslator translator, int maxMemorySize) throws MetaMatrixComponentException { + boolean success = false; + final FileStore lobBuffer = bufferMgr.createFileStore("xml"); //$NON-NLS-1$ try{ - // first persist the XML to the Buffer Manager - TupleBuffer sourceId = createXMLTupleSource(bufferMgr, tupleGroupName); - - // since this object possibly will be streamed we need another streamable - // wrapper layer. - ByteLobChunkStream lobStream = new ByteLobChunkStream(srcXML.getBinaryStream(), chunkSize); - - // read all the globs and Store into storage manager - LobChunk lobChunk = null; - do { - try { - lobChunk = lobStream.getNextChunk(); - } catch (IOException e) { - throw new MetaMatrixComponentException(e); - } - sourceId.addTuple(Arrays.asList(lobChunk)); - } while (!lobChunk.isLast()); - - lobStream.close(); - - sourceId.close(); - return sourceId; - } catch(SQLException e) { - throw new MetaMatrixComponentException(e); + FileStoreOutputStream fsos = lobBuffer.createOutputStream(maxMemorySize); + Writer writer = new OutputStreamWriter(fsos, Streamable.ENCODING); + translator.translate(writer); + writer.close(); + if (!fsos.bytesWritten()) { + return new SQLXMLImpl(fsos.toByteArray()); + } + success = true; + return createSQLXML(lobBuffer); } catch(IOException e) { throw new MetaMatrixComponentException(e); - } - } - - /** - * This will reconstruct the XML object from the buffer manager from given - * buffer manager id. - */ - public static SQLXML getFromBufferManager(final TupleBuffer sourceId, Properties props) { - return new SQLXMLImpl(new BufferMangerXMLReaderFactory(sourceId), props); - } - - /** - * Creates a buffer manager's id for XML based data. - */ - public static TupleBuffer createXMLTupleSource(BufferManager bufferMgr, String tupleGroupName) throws MetaMatrixComponentException { - TupleBuffer sourceID = bufferMgr.createTupleBuffer(getOutputElements(), tupleGroupName, TupleSourceType.PROCESSOR); - sourceID.setBatchSize(1); - return sourceID; - } - - static List getOutputElements() { - ArrayList output = new ArrayList(1); - ElementSymbol xml = new ElementSymbol("xml"); //$NON-NLS-1$ - xml.setType(DataTypeManager.DefaultDataClasses.XML); - output.add(xml); - return output; - } - - /** - * This is buffer manager supported reader factory for the XML lobs - * so that XML can be streamed by the clients more than once. - */ - static class BufferMangerXMLReaderFactory implements XMLReaderFactory { - private TupleBuffer tb; - - BufferMangerXMLReaderFactory(TupleBuffer tb){ - this.tb = tb; + } finally { + if (!success && lobBuffer != null) { + lobBuffer.remove(); + } } - - public Reader getReader() { - final IndexedTupleSource its = tb.createIndexedTupleSource(); - LobChunkProducer producer = new LobChunkProducer() { - public LobChunk getNextChunk() throws IOException { - try { - List<?> tuple = its.nextTuple(); - if (tuple == null) { - return null; - } - return (LobChunk)tuple.get(0); - } catch (MetaMatrixProcessingException e) { - String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", new Object[] {tb}); //$NON-NLS-1$ - LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); - throw new IOException(msg); - } catch (MetaMatrixComponentException e) { - String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.error_processing", new Object[] {tb}); //$NON-NLS-1$ - LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); - throw new IOException(msg); - } - } - - /** - * @see com.metamatrix.common.lob.LobChunkProducer#close() - */ - public void close() throws IOException { - // we could remove the buffer tuple here but, this is just a stream, so we need to delete - // that when we close th eplan. - } - }; - return new LobChunkInputStream(producer).getUTF16Reader(); - } } + + public static SQLXMLImpl createSQLXML(final FileStore lobBuffer) { + SQLXMLImpl sqlXML = new SQLXMLImpl(new InputStreamFactory(Streamable.ENCODING) { + @Override + public InputStream getInputStream() throws IOException { + //TODO: adjust the buffer size, and/or develop a shared buffer strategy + return new BufferedInputStream(lobBuffer.createInputStream()); + } + + @Override + public void free() throws IOException { + lobBuffer.remove(); + } + }); + lobBuffer.setCleanupReference(sqlXML); + return sqlXML; + } + } \ No newline at end of file Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -36,9 +36,9 @@ import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.Streamable; +import com.metamatrix.common.types.XMLTranslator; import com.metamatrix.common.types.XMLType; import com.metamatrix.query.processor.DescribableUtil; import com.metamatrix.query.processor.ProcessorDataManager; @@ -121,7 +121,7 @@ SqlEval sqlEval = new SqlEval(this.dataManager, getContext(), this.xQuery.getProcedureGroup(), this.xQuery.getVariables()); try { - SQLXML xml = expr.evaluateXQuery(sqlEval); + XMLTranslator xml = expr.evaluateXQuery(sqlEval); TupleBatch batch = packResultsIntoBatch(xml); return batch; } finally { @@ -146,19 +146,13 @@ * @param rawResults * @return */ - private TupleBatch packResultsIntoBatch(SQLXML srcXML) throws MetaMatrixComponentException{ + private TupleBatch packResultsIntoBatch(XMLTranslator translator) throws MetaMatrixComponentException{ List rows = new ArrayList(1); List row = new ArrayList(1); - TupleBuffer savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.getContext().getConnectionID(), srcXML, this.chunkSize); + SQLXML srcXML = XMLUtil.saveToBufferManager(this.bufferMgr, translator, this.chunkSize); - //for large documents use the buffermanager version instead - if (savedId.getRowCount() > 1) { - srcXML = XMLUtil.getFromBufferManager(savedId, getFormatProperties()); - } - XMLType xml = new XMLType(srcXML); - savedId.setContainingLobReference(xml); // now build the top batch with information from the saved one. row.add(xml); Modified: trunk/engine/src/main/java/com/metamatrix/query/rewriter/QueryRewriter.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/rewriter/QueryRewriter.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/rewriter/QueryRewriter.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -733,7 +733,7 @@ Iterator iter = outerQuery.getSelect().getProjectedSymbols().iterator(); HashMap<Expression, SingleElementSymbol> expressionMap = new HashMap<Expression, SingleElementSymbol>(); for (SingleElementSymbol symbol : (List<SingleElementSymbol>)query.getSelect().getProjectedSymbols()) { - expressionMap.put((Expression)SymbolMap.getExpression(symbol).clone(), (SingleElementSymbol)iter.next()); + expressionMap.put(SymbolMap.getExpression(symbol), (SingleElementSymbol)iter.next()); } ExpressionMappingVisitor.mapExpressions(groupBy, expressionMap); outerQuery.setGroupBy(groupBy); Modified: trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -298,6 +298,8 @@ for (List<Object> list : tuples) { tsId.addTuple(list); } + //TODO: this leads to fragmented batches, which may require recreating the table + tsId.saveBatch(false); tuplesAdded = tuples.size(); } catch (QueryMetadataException err) { Modified: trunk/engine/src/main/java/com/metamatrix/query/xquery/XQueryExpression.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/xquery/XQueryExpression.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/xquery/XQueryExpression.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,10 +22,9 @@ package com.metamatrix.query.xquery; -import java.sql.SQLXML; - import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; +import com.metamatrix.common.types.XMLTranslator; /** * An XQueryExpression - the object representation of @@ -53,11 +52,10 @@ * value is interpreted to mean that XQueries are not supported * by this engine. * @param compiledXQuery compiled XQueryExpression - * @return retuns a result SQLXML object. * @throws MetaMatrixProcessingException if xQueryString is * invalid and fails to compile */ - public SQLXML evaluateXQuery(XQuerySQLEvaluator sqlEval) + public XMLTranslator evaluateXQuery(XQuerySQLEvaluator sqlEval) throws MetaMatrixProcessingException, MetaMatrixComponentException; Deleted: trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXMLTranslator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXMLTranslator.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXMLTranslator.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,117 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.xquery.saxon; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.Properties; - -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.TransformerException; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; - -import net.sf.saxon.Configuration; -import net.sf.saxon.om.NodeInfo; -import net.sf.saxon.query.QueryResult; - -import com.metamatrix.common.types.XMLTranslator; - - -/** - * This converts the SAXOn based XML tree into another popular XMl formats like String, - * DOM, SAX etc. - */ -public class SaxonXMLTranslator implements XMLTranslator { - NodeInfo source; - Properties properties; - - public SaxonXMLTranslator(NodeInfo source, Properties props) { - this.source = source; - this.properties = props; - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getString() - */ - public String getString() throws IOException { - StringWriter writer = new StringWriter(); - Result result = new StreamResult(writer); - try { - QueryResult.serialize(source, result, this.properties, new Configuration()); - } catch (TransformerException e) { - throw new IOException(e.getMessage()); - } - StringBuffer buffer = writer.getBuffer(); - return buffer.toString(); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getSource() - */ - public Source getSource() throws IOException { - return getStreamSource(); - } - - /** - * Get a Stream source from the SAXON's tiny tree format. - */ - private StreamSource getStreamSource() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Result result = new StreamResult(out); - try { - QueryResult.serialize(source, result, this.properties, new Configuration()); - } catch (TransformerException e) { - throw new IOException(e.getMessage()); - } - out.close(); - return new StreamSource(new ByteArrayInputStream(out.toByteArray())); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getReader() - */ - public Reader getReader() throws IOException { - return new StringReader(getString()); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getInputStream() - */ - public InputStream getInputStream() throws IOException { - return getStreamSource().getInputStream(); - } - - /** - * @see com.metamatrix.common.types.XMLTranslator#getProperties() - */ - public Properties getProperties() { - return this.properties; - } -} Modified: trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,6 +22,8 @@ package com.metamatrix.query.xquery.saxon; +import java.io.IOException; +import java.io.Writer; import java.sql.SQLException; import java.sql.SQLXML; import java.util.HashMap; @@ -34,16 +36,18 @@ import javax.xml.transform.ErrorListener; import javax.xml.transform.TransformerException; +import javax.xml.transform.stream.StreamResult; import net.sf.saxon.Configuration; import net.sf.saxon.om.NodeInfo; import net.sf.saxon.query.DynamicQueryContext; +import net.sf.saxon.query.QueryResult; import net.sf.saxon.query.StaticQueryContext; import net.sf.saxon.trans.XPathException; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.types.SQLXMLImpl; +import com.metamatrix.common.types.XMLTranslator; import com.metamatrix.query.QueryPlugin; import com.metamatrix.query.util.XMLFormatConstants; import com.metamatrix.query.xquery.XQueryExpression; @@ -117,7 +121,7 @@ /** * @see com.metamatrix.query.xquery.XQueryEngine#evaluateXQuery(com.metamatrix.query.xquery.XQueryExpression) */ - public SQLXML evaluateXQuery(XQuerySQLEvaluator sqlEval) + public XMLTranslator evaluateXQuery(XQuerySQLEvaluator sqlEval) throws MetaMatrixProcessingException, MetaMatrixComponentException { Configuration config = new Configuration(); @@ -172,11 +176,22 @@ // output if (obj instanceof NodeInfo){ - Properties props = new Properties(); + final NodeInfo nodeInfo = (NodeInfo)obj; + final Properties props = new Properties(); if (XMLFormatConstants.XML_TREE_FORMAT.equals(this.xmlFormat)) { props.setProperty("indent", "yes");//$NON-NLS-1$//$NON-NLS-2$ } - return new SQLXMLImpl(new SaxonXMLTranslator((NodeInfo)obj, props)); + return new XMLTranslator() { + + @Override + public void translate(Writer writer) throws IOException { + try { + QueryResult.serialize(nodeInfo, new StreamResult(writer), props, new Configuration()); + } catch (TransformerException e) { + throw new IOException(e); + } + } + }; } } throw new MetaMatrixProcessingException(QueryPlugin.Util.getString("wrong_result_type")); //$NON-NLS-1$ Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -58,6 +58,7 @@ import com.metamatrix.common.log.LogManager; import com.metamatrix.common.queue.WorkerPool; import com.metamatrix.common.queue.WorkerPoolFactory; +import com.metamatrix.common.types.Streamable; import com.metamatrix.common.util.PropertiesUtils; import com.metamatrix.common.xa.MMXid; import com.metamatrix.common.xa.XATransactionException; @@ -153,7 +154,7 @@ private int processorTimeslice = DEFAULT_PROCESSOR_TIMESLICE; private boolean processorDebugAllowed; - private int chunkSize = 0; + private int chunkSize = Streamable.STREAMING_BATCH_SIZE_IN_BYTES; private Map<RequestID, RequestWorkItem> requests = new ConcurrentHashMap<RequestID, RequestWorkItem>(); private Map<String, ClientState> clientState = Collections.synchronizedMap(new HashMap<String, ClientState>()); @@ -381,8 +382,6 @@ } return getQueueStatistics(); } - - /** * Cancel and close all requests associated with the clientConnection/session. Also runs a final cleanup any caches within @@ -407,13 +406,6 @@ } } - // cleanup the buffer manager - try { - bufferManager.removeTupleBuffers(sessionId); - } catch (Exception e) { - LogManager.logWarning(LogConstants.CTX_DQP, e, "Failed to remove buffered tuples for connection " + sessionId); //$NON-NLS-1$ - } - if (transactionService != null) { try { transactionService.cancelTransactions(sessionId, false); @@ -568,7 +560,6 @@ return chunkSize; } - /* * @see com.metamatrix.common.application.Application#initialize(java.util.Properties) */ @@ -616,7 +607,7 @@ this.maxCodeTables = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.MAX_CODE_TABLES, DEFAULT_MAX_CODE_TABLES); this.maxCodeRecords = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.MAX_CODE_TABLE_RECORDS, DEFAULT_MAX_CODE_RECORDS); - this.chunkSize = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.STREAMING_BATCH_SIZE, 10) * 1024; + this.chunkSize = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.STREAMING_BATCH_SIZE, 100) * 1024; //result set cache /*if(PropertiesUtils.getBooleanProperty(props, DQPEmbeddedProperties.USE_RESULTSET_CACHE, false)){ Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -38,15 +38,11 @@ import com.metamatrix.common.types.XMLType; import com.metamatrix.core.util.Assertion; import com.metamatrix.dqp.DQPPlugin; -import com.metamatrix.dqp.message.RequestID; import com.metamatrix.dqp.util.LogConstants; public class LobWorkItem implements Runnable { private RequestWorkItem parent; - private RequestID requestID; - - private DQPCore dqpCore; private int chunkSize; /* private work item state */ @@ -57,8 +53,6 @@ public LobWorkItem(RequestWorkItem parent, DQPCore dqpCore, String streamId, int streamRequestId) { this.chunkSize = dqpCore.getChunkSize(); - this.dqpCore = dqpCore; - this.requestID = parent.requestID; this.streamId = streamId; this.parent = parent; this.streamRequestId = streamRequestId; @@ -122,18 +116,16 @@ try { if (streamable instanceof XMLType) { XMLType xml = (XMLType)streamable; - return new ByteLobChunkStream(new ReaderInputStream(xml.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$ + return new ByteLobChunkStream(new ReaderInputStream(xml.getCharacterStream(), Charset.forName(Streamable.ENCODING)), chunkSize); } else if (streamable instanceof ClobType) { ClobType clob = (ClobType)streamable; - return new ByteLobChunkStream(new ReaderInputStream(clob.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$ + return new ByteLobChunkStream(new ReaderInputStream(clob.getCharacterStream(), Charset.forName(Streamable.ENCODING)), chunkSize); } BlobType blob = (BlobType)streamable; return new ByteLobChunkStream(blob.getBinaryStream(), chunkSize); } catch(SQLException e) { - IOException ex = new IOException(); - ex.initCause(e); - throw ex; + throw new IOException(e); } } Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -94,7 +94,7 @@ final DQPWorkContext dqpWorkContext; //results request - ResultsReceiver<ResultsMessage> resultsReceiver; + private ResultsReceiver<ResultsMessage> resultsReceiver; private int begin; private int end; @@ -366,70 +366,72 @@ protected void sendResultsIfNeeded(TupleBatch batch) throws MetaMatrixComponentException { ResultsMessage response = null; ResultsReceiver<ResultsMessage> receiver = null; + synchronized (this) { if (this.resultsReceiver == null || (this.begin > this.processor.getHighestRow() && !doneProducingBatches) || (this.transactionState == TransactionState.ACTIVE)) { return; } - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID: " + requestID + " resultsID: " + this.resultsBuffer + " done: " + doneProducingBatches ); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - - if (batch == null || batch.getBeginRow() > this.begin) { - batch = resultsBuffer.getBatch(begin); - //TODO: support fetching more than 1 batch - int count = this.end - this.begin + 1; - if (batch.getRowCount() > count) { - int beginRow = Math.min(this.begin, batch.getEndRow() - count + 1); - int endRow = Math.min(beginRow + count - 1, batch.getEndRow()); - int firstOffset = beginRow - batch.getBeginRow(); - List[] memoryRows = batch.getAllTuples(); - List[] rows = new List[count]; - System.arraycopy(memoryRows, firstOffset, rows, 0, endRow - beginRow + 1); - batch = new TupleBatch(beginRow, rows); - } - } - int finalRowCount = doneProducingBatches?this.processor.getHighestRow():-1; - - response = createResultsMessage(requestMsg, batch.getAllTuples(), this.processor.getProcessorPlan().getOutputElements(), analysisRecord); - response.setFirstRow(batch.getBeginRow()); - response.setLastRow(batch.getEndRow()); - response.setUpdateResult(this.returnsUpdateCount); - // set final row - response.setFinalRow(finalRowCount); - - // send any schemas associated with the results - response.setSchemas(this.schemas); - - // send any warnings with the response object - List<Throwable> responseWarnings = new ArrayList<Throwable>(); - List<Exception> currentWarnings = processor.getAndClearWarnings(); - if (currentWarnings != null) { - responseWarnings.addAll(currentWarnings); - } - synchronized (warnings) { - responseWarnings.addAll(this.warnings); - this.warnings.clear(); - } - response.setWarnings(responseWarnings); - - // If it is stored procedure, set parameters - if (originalCommand instanceof StoredProcedure) { - StoredProcedure proc = (StoredProcedure)originalCommand; - if (proc.returnParameters()) { - response.setParameters(getParameterInfo(proc)); - } - } - /* - * mark the results sent at this point. - * communication exceptions will be treated as non-recoverable - */ - receiver = this.resultsReceiver; - this.resultsReceiver = null; } + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID: " + requestID + " resultsID: " + this.resultsBuffer + " done: " + doneProducingBatches ); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + if (batch == null || batch.getBeginRow() > this.begin) { + batch = resultsBuffer.getBatch(begin); + //TODO: support fetching more than 1 batch + int count = this.end - this.begin + 1; + if (batch.getRowCount() > count) { + int beginRow = Math.min(this.begin, batch.getEndRow() - count + 1); + int endRow = Math.min(beginRow + count - 1, batch.getEndRow()); + int firstOffset = beginRow - batch.getBeginRow(); + List[] memoryRows = batch.getAllTuples(); + List[] rows = new List[count]; + System.arraycopy(memoryRows, firstOffset, rows, 0, endRow - beginRow + 1); + batch = new TupleBatch(beginRow, rows); + } + } + int finalRowCount = doneProducingBatches?this.processor.getHighestRow():-1; + + response = createResultsMessage(requestMsg, batch.getAllTuples(), this.processor.getProcessorPlan().getOutputElements(), analysisRecord); + response.setFirstRow(batch.getBeginRow()); + response.setLastRow(batch.getEndRow()); + response.setUpdateResult(this.returnsUpdateCount); + // set final row + response.setFinalRow(finalRowCount); + + // send any schemas associated with the results + response.setSchemas(this.schemas); + + // send any warnings with the response object + List<Throwable> responseWarnings = new ArrayList<Throwable>(); + List<Exception> currentWarnings = processor.getAndClearWarnings(); + if (currentWarnings != null) { + responseWarnings.addAll(currentWarnings); + } + synchronized (warnings) { + responseWarnings.addAll(this.warnings); + this.warnings.clear(); + } + response.setWarnings(responseWarnings); + + // If it is stored procedure, set parameters + if (originalCommand instanceof StoredProcedure) { + StoredProcedure proc = (StoredProcedure)originalCommand; + if (proc.returnParameters()) { + response.setParameters(getParameterInfo(proc)); + } + } + /* + * mark the results sent at this point. + * communication exceptions will be treated as non-recoverable + */ + synchronized (this) { + receiver = this.resultsReceiver; + this.resultsReceiver = null; + } + receiver.receiveResults(response); } Deleted: trunk/engine/src/test/java/com/metamatrix/common/buffer/TestTupleSourceID.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/TestTupleSourceID.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/TestTupleSourceID.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -1,298 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -import junit.framework.TestCase; - -import com.metamatrix.core.util.UnitTestUtil; - -/** - */ -public class TestTupleSourceID extends TestCase { - - private static final String ID_VALUE = "idValue"; //$NON-NLS-1$ - private static final String LOCATION_VALUE = "location"; //$NON-NLS-1$ - private TupleSourceID objWithIdAndLocation; - private TupleSourceID objWithId; - private TupleSourceID objWithLocation; - - /** - * Constructor for TestTupleSourceID. - * @param name - */ - public TestTupleSourceID(String name) { - super(name); - } - - /** - * @see TestCase#setUp() - */ - protected void setUp() throws Exception { - super.setUp(); - this.objWithId = new TupleSourceID(ID_VALUE,null); - this.objWithIdAndLocation = new TupleSourceID(ID_VALUE,LOCATION_VALUE); - this.objWithLocation = new TupleSourceID("",LOCATION_VALUE); //$NON-NLS-1$ - } - - /** - * @see TestCase#tearDown() - */ - protected void tearDown() throws Exception { - super.tearDown(); - } - - // ========================================================================= - // H E L P E R M E T H O D S - // ========================================================================= - - public TupleSourceID helpTestConstructor( final String location, final String id, - final boolean shouldSucceed ) { - try { - final TupleSourceID channelId = new TupleSourceID(location,id); - // Make sure that we were supposed to have succeeded - if ( !shouldSucceed ) { - fail("Did not expect to construct successfully"); //$NON-NLS-1$ - } - return channelId; - } catch ( Throwable t ) { - if ( shouldSucceed ) { - throw new RuntimeException(t); - } - } - return null; - } - - public TupleSourceID helpTestConstructor( final String idValue, - final boolean shouldSucceed ) { - try { - final TupleSourceID channelId = new TupleSourceID(idValue); - // Make sure that we were supposed to have succeeded - if ( !shouldSucceed ) { - fail("Did not expect to construct successfully"); //$NON-NLS-1$ - } - return channelId; - } catch ( Throwable t ) { - if ( shouldSucceed ) { - throw new RuntimeException(t); - } - } - return null; - } - - public void helpCheckStringId( final TupleSourceID id, final String expected ) { - final String actual = id.getStringID(); - assertEquals(expected, actual); - } - - public void helpCheckId( final TupleSourceID id, final String expected ) { - final String actual = id.getIDValue(); - assertEquals(expected, actual); - } - - public void helpCheckLocation( final TupleSourceID id, final String expected ) { - final String actual = id.getLocation(); - assertEquals(expected, actual); - } - - // ========================================================================= - // T E S T C A S E S - // ========================================================================= - - /* - * Test for void TupleSourceID(String) - */ - public void testTupleSourceIDStringWithIdAndNoLocation() { - final String idValue = "SingleIdWithoutDelimiter"; //$NON-NLS-1$ - final TupleSourceID id = helpTestConstructor(idValue, true); - helpCheckId(id,idValue); - helpCheckStringId(id, idValue); - helpCheckLocation(id, null); - } - - /* - * Test for void TupleSourceID(String, String) - */ - public void testTupleSourceIDStringWithIdAndLocation() { - final String idValue = "SingleId"; //$NON-NLS-1$ - final String location = "Location"; //$NON-NLS-1$ - final String stringId = location + ":" + idValue; //$NON-NLS-1$ - final TupleSourceID id = helpTestConstructor(stringId, true); - helpCheckId(id,idValue); - helpCheckStringId(id, stringId); - helpCheckLocation(id, location); - } - - /* - * Test for void TupleSourceID(String, String) - */ - public void testTupleSourceIDStringWithNullIdAndLocation() { - final String idValue = ""; //$NON-NLS-1$ - final String location = "Location"; //$NON-NLS-1$ - final String stringId = location + ":" + idValue; //$NON-NLS-1$ - final TupleSourceID id = helpTestConstructor(stringId, true); - helpCheckId(id,idValue); - helpCheckStringId(id, stringId); - helpCheckLocation(id, location); - } - - /* - * Test for void TupleSourceID(String, String) - */ - public void testTupleSourceIDStringWithIdAndNullLocation() { - final String idValue = "SingleId"; //$NON-NLS-1$ - final String location = ""; //$NON-NLS-1$ - final String stringId = location + ":" + idValue; //$NON-NLS-1$ - final TupleSourceID id = helpTestConstructor(stringId, true); - helpCheckId(id,idValue); - helpCheckStringId(id, stringId); - helpCheckLocation(id, location); - } - - public void testIllegalTupleSourceIDString() { - final String stringId = null; - helpTestConstructor(stringId, false); - } - - public void testHashCode1() { - assertEquals(objWithId.hashCode(), ID_VALUE.hashCode()); - } - - public void testHashCode2() { - assertEquals(objWithLocation.hashCode(), "".hashCode()); //$NON-NLS-1$ - } - - public void testHashCode3() { - assertEquals(objWithIdAndLocation.hashCode(), ID_VALUE.hashCode()); - } - - public void testGetLocation1() { - assertEquals(objWithId.getLocation(), null); - } - - public void testGetLocation2() { - assertEquals(objWithLocation.getLocation(), LOCATION_VALUE); - } - - public void testGetLocation3() { - assertEquals(objWithIdAndLocation.getLocation(), LOCATION_VALUE); - } - - public void testGetIDValue1() { - assertEquals(objWithId.getIDValue(), ID_VALUE); - } - - public void testGetIDValue2() { - assertEquals(objWithLocation.getIDValue(), ""); //$NON-NLS-1$ - } - - public void testGetIDValue3() { - assertEquals(objWithIdAndLocation.getIDValue(), ID_VALUE); - } - - public void testGetStringID1() { - assertEquals(objWithId.getStringID(), ID_VALUE); - } - - public void testGetStringID2() { - assertEquals(objWithLocation.getStringID(), LOCATION_VALUE + ":"); //$NON-NLS-1$ - } - - public void testGetStringID3() { - assertEquals(objWithIdAndLocation.getStringID(), LOCATION_VALUE + ":" + ID_VALUE); //$NON-NLS-1$ - } - - /* - * Test for boolean equals(Object) - */ - public void testEqualsSameObject1() { - assertTrue(objWithId.equals(objWithId)); - } - - public void testEqualsSameObject2() { - assertTrue(objWithLocation.equals(objWithLocation)); - } - - public void testEqualsSameObject3() { - assertTrue(objWithIdAndLocation.equals(objWithIdAndLocation)); - } - - public void testEqualsSimilarObject1() { - assertTrue(objWithId.equals(new TupleSourceID(ID_VALUE, null))); - } - - public void testEqualsSimilarObject2() { - assertTrue(objWithLocation.equals(new TupleSourceID("", LOCATION_VALUE))); //$NON-NLS-1$ - } - - public void testEqualsSimilarObject3() { - assertTrue(objWithIdAndLocation.equals(new TupleSourceID(ID_VALUE, LOCATION_VALUE))); - } - - /* - * Test for String toString() - */ - public void testToString1() { - assertEquals(objWithId.toString(), ID_VALUE); - } - - public void testToString2() { - assertEquals(objWithLocation.toString(), LOCATION_VALUE + ":"); //$NON-NLS-1$ - } - - public void testToString3() { - assertEquals(objWithIdAndLocation.toString(), LOCATION_VALUE + ":" + ID_VALUE); //$NON-NLS-1$ - } - - public void testRoundtrip1() { - TupleSourceID expectedID = new TupleSourceID("1", "mymachine,100"); //$NON-NLS-1$ //$NON-NLS-2$ - String stringID = expectedID.getStringID(); - TupleSourceID actualID = new TupleSourceID(stringID); - - assertEquals("Different ID after roundtrip: ", expectedID.getIDValue(), actualID.getIDValue()); //$NON-NLS-1$ - assertEquals("Different location after roundtrip: ", expectedID.getIDValue(), actualID.getIDValue()); //$NON-NLS-1$ - } - - public void testRoundtrip2() { - TupleSourceID expectedID = new TupleSourceID("1"); //$NON-NLS-1$ - String stringID = expectedID.getStringID(); - TupleSourceID actualID = new TupleSourceID(stringID); - - assertEquals("Different ID after roundtrip: ", expectedID.getIDValue(), actualID.getIDValue()); //$NON-NLS-1$ - assertEquals("Different location after roundtrip: ", expectedID.getIDValue(), actualID.getIDValue()); //$NON-NLS-1$ - } - - public void testStandard1() { - UnitTestUtil.helpTestEquivalence(0, new TupleSourceID("1"), new TupleSourceID("1")); //$NON-NLS-1$ //$NON-NLS-2$ - } - - public void testStandard2() { - UnitTestUtil.helpTestEquivalence(0, new TupleSourceID("1"), new TupleSourceID("1", "mymachine,100")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - - public void testNotEquals() { - TupleSourceID id1 = new TupleSourceID("1"); //$NON-NLS-1$ - TupleSourceID id2 = new TupleSourceID("2"); //$NON-NLS-1$ - assertTrue("Differing IDs compare as equal", ! id1.equals(id2)); //$NON-NLS-1$ - } -} - Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestAggregateProcessing.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestAggregateProcessing.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestAggregateProcessing.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -244,28 +244,25 @@ dataManager.addData("SELECT v_0.c_0, COUNT(v_0.c_1), MAX(v_0.c_1) FROM (SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM pm1.g1 AS g_0) AS v_0 GROUP BY v_0.c_0", //$NON-NLS-1$ new List[] { - Arrays.asList("a", Integer.valueOf(2), Boolean.TRUE), //$NON-NLS-1$ - Arrays.asList("b", null, null) //$NON-NLS-1$ + Arrays.asList("a", Integer.valueOf(2), Integer.valueOf(1)), //$NON-NLS-1$ }); dataManager.addData("SELECT v_0.c_0, COUNT(v_0.c_1), MAX(v_0.c_1) FROM (SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM pm1.g2 AS g_0) AS v_0 GROUP BY v_0.c_0", //$NON-NLS-1$ new List[] { - Arrays.asList("a", Integer.valueOf(3), Boolean.FALSE), //$NON-NLS-1$ - Arrays.asList("b", null, null) //$NON-NLS-1$ + Arrays.asList("a", Integer.valueOf(3), Integer.valueOf(2)), //$NON-NLS-1$ }); dataManager.addData("SELECT g_0.e1, g_0.e2 FROM pm2.g1 AS g_0", //$NON-NLS-1$ new List[] { - Arrays.asList("a", Boolean.FALSE), //$NON-NLS-1$ - Arrays.asList("xyz", Boolean.FALSE), //$NON-NLS-1$ - Arrays.asList(null, Boolean.TRUE), + Arrays.asList("a", Integer.valueOf(3)), //$NON-NLS-1$ + Arrays.asList("xyz", Integer.valueOf(4)), //$NON-NLS-1$ + Arrays.asList(null, Integer.valueOf(5)), }); ProcessorPlan plan = helpGetPlan(command, FakeMetadataFactory.example1Cached(), capFinder); List[] expected = new List[] { - Arrays.asList(null, Integer.valueOf(1), Boolean.TRUE), - Arrays.asList("a", Integer.valueOf(6), Boolean.TRUE), //$NON-NLS-1$ - Arrays.asList("b", null, null), //$NON-NLS-1$ - Arrays.asList("xyz", Integer.valueOf(1), Boolean.FALSE) //$NON-NLS-1$ + Arrays.asList(null, Integer.valueOf(1), Integer.valueOf(5)), + Arrays.asList("a", Integer.valueOf(6), Integer.valueOf(3)), //$NON-NLS-1$ + Arrays.asList("xyz", Integer.valueOf(1), Integer.valueOf(4)) //$NON-NLS-1$ }; helpProcess(plan, dataManager, expected); @@ -283,12 +280,12 @@ dataManager.addData("SELECT v_0.c_1, v_0.c_0, COUNT(*), MAX(v_0.c_2) FROM (SELECT g_0.e2 AS c_0, g_0.e1 AS c_1, g_0.e3 AS c_2 FROM pm1.g1 AS g_0) AS v_0 GROUP BY v_0.c_0, v_0.c_1", //$NON-NLS-1$ new List[] { - Arrays.asList(Integer.valueOf(2), "2", Integer.valueOf(2), Boolean.FALSE), //$NON-NLS-1$ - Arrays.asList(Integer.valueOf(1), "1", Integer.valueOf(3), Boolean.TRUE), //$NON-NLS-1$ + Arrays.asList("2", Integer.valueOf(2), Integer.valueOf(2), Boolean.FALSE), //$NON-NLS-1$ + Arrays.asList("1", Integer.valueOf(1), Integer.valueOf(3), Boolean.TRUE), //$NON-NLS-1$ }); dataManager.addData("SELECT v_0.c_1, v_0.c_0, COUNT(*), MAX(v_0.c_2) FROM (SELECT g_0.e2 AS c_0, convert(g_0.e2, string) AS c_1, g_0.e3 AS c_2 FROM pm2.g2 AS g_0 ORDER BY c_1 LIMIT 10) AS v_0 GROUP BY v_0.c_0, v_0.c_1", //$NON-NLS-1$ new List[] { - Arrays.asList(Integer.valueOf(1), "1", Integer.valueOf(4), Boolean.FALSE), //$NON-NLS-1$ + Arrays.asList("1", Integer.valueOf(1), Integer.valueOf(4), Boolean.FALSE), //$NON-NLS-1$ }); ProcessorPlan plan = helpGetPlan(command, FakeMetadataFactory.example1Cached(), capFinder); Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -7543,6 +7543,21 @@ sampleData1(manager); helpProcess(plan, manager, expected); } + + @Test public void testOrderByDescAll() { + String sql = "SELECT distinct e1 from pm1.g2 order by e1 desc limit 1"; //$NON-NLS-1$ + + List[] expected = new List[] { + Arrays.asList("c"), + }; + + FakeDataManager dataManager = new FakeDataManager(); + sampleData1(dataManager); + + ProcessorPlan plan = helpGetPlan(helpParse(sql), FakeMetadataFactory.example1Cached()); + + helpProcess(plan, dataManager, expected); + } private static final boolean DEBUG = false; } Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -39,7 +39,6 @@ import com.metamatrix.api.exception.MetaMatrixException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleBuffer; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.buffer.impl.BufferManagerImpl; import com.metamatrix.common.buffer.impl.MemoryStorageManager; import com.metamatrix.common.types.DataTypeManager; @@ -154,7 +153,7 @@ FakeMetadataObject accounts = FakeMetadataFactory.createVirtualGroup("Accounts.Accounts", vAccts, accountsPlan); //$NON-NLS-1$ List accountsElem = FakeMetadataFactory.createElements(accounts, new String[] { "customer_id", "account_id", "transaction_id", "txn_type", "amount", "source" }, //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ - new String[] { DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.BIG_DECIMAL, DataTypeManager.DefaultDataTypes.STRING }); + new String[] { DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.BIG_DECIMAL, DataTypeManager.DefaultDataTypes.STRING }); FakeMetadataObject master = FakeMetadataFactory.createVirtualModel("Master"); //$NON-NLS-1$ QueryNode masterPlan = new QueryNode("Master.Transactions", //$NON-NLS-1$ @@ -162,7 +161,7 @@ FakeMetadataObject transactions = FakeMetadataFactory.createVirtualGroup("Master.Transactions", master, masterPlan); //$NON-NLS-1$ List transactionsElem = FakeMetadataFactory.createElements(transactions, new String[] { "CustomerID", "First", "Last", "AccountID", "TransactionID", "TxnCode", "Amount" }, //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ - new String[] { DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.STRING, DataTypeManager.DefaultDataTypes.STRING, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.STRING }); + new String[] { DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.STRING, DataTypeManager.DefaultDataTypes.STRING, DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.LONG, DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataTypes.BIG_DECIMAL }); FakeMetadataStore store = new FakeMetadataStore(); store.addObject(us); @@ -418,7 +417,6 @@ private void helpTestMultipleBatches(boolean unique) throws Exception, MetaMatrixComponentException, MetaMatrixException, - TupleSourceNotFoundException, SQLException { // Create query String sql = "SELECT * from Master.Transactions where last = 'Davis' order by CustomerID, TransactionID"; //$NON-NLS-1$ Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -22,23 +22,15 @@ package com.metamatrix.query.processor.dynamic; -import java.io.BufferedReader; import java.util.Collections; import javax.xml.transform.Source; -import javax.xml.transform.sax.SAXSource; -import javax.xml.transform.stream.StreamSource; import junit.framework.TestCase; -import org.xml.sax.Attributes; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; -import org.xml.sax.SAXParseException; -import org.xml.sax.helpers.DefaultHandler; - import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerFactory; +import com.metamatrix.common.types.StandardXMLTranslator; import com.metamatrix.query.metadata.QueryMetadataInterface; import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.capabilities.DefaultCapabilitiesFinder; @@ -77,55 +69,15 @@ } String toXMLString(Source xmlSrc) throws Exception{ - if (xmlSrc instanceof StreamSource) { - StreamSource input = (StreamSource)xmlSrc; - BufferedReader reader = new BufferedReader(input.getReader()); - StringBuffer sb = new StringBuffer(); - - int line = reader.read(); - while(line != -1) { - sb.append((char)line); - line = reader.read(); - } - reader.close(); - return sb.toString(); - } - else if (xmlSrc instanceof SAXSource) { - SAXSource input = (SAXSource)xmlSrc; - final StringBuffer sb = new StringBuffer(); - ContentHandler handler = new DefaultHandler() { - - public void characters(char[] ch,int start,int length) throws SAXException { - sb.append(ch, start, length); - } - public void endDocument() throws SAXException { - super.endDocument(); - } - public void endElement(String uri,String localName,String qName) throws SAXException { - sb.append("</").append(qName).append(">"); //$NON-NLS-1$ //$NON-NLS-2$ - } - public void error(SAXParseException e) throws SAXException { - super.error(e); - } - public void startDocument() throws SAXException { - super.startDocument(); - } - public void startElement(String uri,String localName,String qName, Attributes attributes) throws SAXException { - sb.append("<").append(qName).append(">"); //$NON-NLS-1$ //$NON-NLS-2$ - } - }; - input.getXMLReader().setContentHandler(handler); - input.getXMLReader().parse(input.getInputSource()); - return sb.toString(); - } - return ""; //$NON-NLS-1$ + StandardXMLTranslator sxt = new StandardXMLTranslator(xmlSrc, null); + return sxt.getString(); } public void testTableResult() throws Exception { // Create query String sql = "SELECT pm1.g1.e1, e2, pm1.g1.e3 as a, e4 as b FROM pm1.g1"; //$NON-NLS-1$ - String expected ="" + //$NON-NLS-1$ + String expected ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + //$NON-NLS-1$ "<results>" + //$NON-NLS-1$ "<row><e1>a</e1><e2>0</e2><a>false</a><b>2.0</b></row>" + //$NON-NLS-1$ "<row><e1>null</e1><e2>1</e2><a>false</a><b>1.0</b></row>" +//$NON-NLS-1$ @@ -148,9 +100,8 @@ // Create query String sql = "SELECT pm1.g1.e1, e2, pm1.g1.e3 as a, e4 as b FROM pm1.g1 where e2=4"; //$NON-NLS-1$ - String expected ="" + //$NON-NLS-1$ - "<results>" + //$NON-NLS-1$ - "</results>";//$NON-NLS-1$ + String expected ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + //$NON-NLS-1$ + "<results/>"; //$NON-NLS-1$ // Construct data manager with data FakeDataManager dataMgr = new FakeDataManager(); @@ -166,7 +117,7 @@ FakeMetadataFacade metadata = TestXMLProcessor.exampleMetadataCached(); FakeDataManager dataMgr = TestXMLProcessor.exampleDataManager(metadata); String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + //$NON-NLS-1$ + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + //$NON-NLS-1$ "<Catalogs xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n&q... + //$NON-NLS-1$ " <Catalog>\n" + //$NON-NLS-1$ " <Items>\n" + //$NON-NLS-1$ Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -42,7 +42,6 @@ import com.metamatrix.common.buffer.BufferManagerFactory; import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.XMLType; import com.metamatrix.core.MetaMatrixCoreException; Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestGroupingNode.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestGroupingNode.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestGroupingNode.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -64,10 +64,11 @@ // ################################## TEST HELPERS ################################ public static TupleSource createTupleSource1() { - List symbols = new ArrayList(); + List<ElementSymbol> symbols = new ArrayList<ElementSymbol>(); symbols.add(new ElementSymbol("col1")); //$NON-NLS-1$ + symbols.get(0).setType(DataTypeManager.DefaultDataClasses.INTEGER); symbols.add(new ElementSymbol("col2")); //$NON-NLS-1$ - + symbols.get(1).setType(DataTypeManager.DefaultDataClasses.INTEGER); List[] tuples = new List[] { Arrays.asList(new Object[] { new Integer(5), new Integer(3) }), Arrays.asList(new Object[] { new Integer(2), new Integer(1) }), @@ -158,7 +159,7 @@ node.setElements(outputElements); List groupingElements = new ArrayList(); - groupingElements.add(new ElementSymbol("col1")); //$NON-NLS-1$ + groupingElements.add(col1); //$NON-NLS-1$ node.setGroupingElements(groupingElements); CommandContext context = new CommandContext("pid", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ node.initialize(context, mgr, null); @@ -281,7 +282,7 @@ // Set grouping elements to null List groupingElements = new ArrayList(); - groupingElements.add(new ElementSymbol("col1")); //$NON-NLS-1$ + groupingElements.add(col1); //$NON-NLS-1$ node.setGroupingElements(groupingElements); CommandContext context = new CommandContext("pid", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ node.initialize(context, mgr, null); @@ -330,7 +331,7 @@ node.setElements(outputElements); List groupingElements = new ArrayList(); - groupingElements.add(new ElementSymbol("col1")); //$NON-NLS-1$ + groupingElements.add(col1); //$NON-NLS-1$ node.setGroupingElements(groupingElements); CommandContext context = new CommandContext("pid", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ @@ -448,7 +449,7 @@ node.setElements(outputElements); List groupingElements = new ArrayList(); - groupingElements.add(new ElementSymbol("col1")); //$NON-NLS-1$ + groupingElements.add(col1); //$NON-NLS-1$ node.setGroupingElements(groupingElements); return node; } Modified: trunk/engine/src/test/java/com/metamatrix/query/xquery/TestXQueryEngine.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/xquery/TestXQueryEngine.java 2010-01-08 19:50:03 UTC (rev 1722) +++ trunk/engine/src/test/java/com/metamatrix/query/xquery/TestXQueryEngine.java 2010-01-08 19:51:25 UTC (rev 1723) @@ -23,6 +23,9 @@ package com.metamatrix.query.xquery; import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; import java.net.URI; import java.sql.SQLXML; import java.util.HashMap; @@ -35,6 +38,7 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.types.SQLXMLImpl; +import com.metamatrix.common.types.XMLTranslator; import com.metamatrix.core.util.FileUtil; import com.metamatrix.core.util.UnitTestUtil; import com.metamatrix.query.xquery.saxon.SaxonXQueryExpression; @@ -61,8 +65,9 @@ private void helpTestEngine(XQueryExpression expr, String xQuery, SQLXML expected, XQuerySQLEvaluator sqlEval) throws Exception{ expr.compileXQuery(xQuery); - SQLXML actualResults = expr.evaluateXQuery(sqlEval); - assertEquals(expected.getString(), actualResults.getString()); + XMLTranslator actualResults = expr.evaluateXQuery(sqlEval); + String result = actualResults.getString(); + assertEquals(expected.getString(), result); } private void helpTestEngineFails(String xQuery, Class<?> expectedFailure, XQuerySQLEvaluator sqlEval) throws Exception{ @@ -154,7 +159,7 @@ XQueryExpression expr = new SaxonXQueryExpression(); expr.compileXQuery(xquery); - SQLXML actualResults = expr.evaluateXQuery(new HardcodedSqlEval(null)); + XMLTranslator actualResults = expr.evaluateXQuery(new HardcodedSqlEval(null)); assertEquals(expectedStr, actualResults.getString().replaceAll("[ \t\r\n]", "")); //$NON-NLS-1$ //$NON-NLS-2$ }