August 2009 - teiid-commits - Jboss List Archives

teiid SVN: r1235 - in trunk: common-core/src/main/java/com/metamatrix/common/types and 20 other directories.

by teiid-commits＠lists.jboss.org

Author: shawkins Date: 2009-08-12 13:39:14 -0400 (Wed, 12 Aug 2009) New Revision: 1235 Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java Removed: trunk/common-core/src/main/java/com/metamatrix/common/types/InvalidReferenceException.java trunk/engine/src/main/java/com/metamatrix/common/buffer/LobTupleBatch.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryState.java trunk/engine/src/main/java/com/metamatrix/common/lob/ Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMBlob.java trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java trunk/common-core/src/main/java/com/metamatrix/common/types/BlobType.java trunk/common-core/src/main/java/com/metamatrix/common/types/ClobType.java trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java trunk/common-core/src/main/java/com/metamatrix/common/types/XMLType.java trunk/common-core/src/main/java/com/metamatrix/common/types/basic/ClobToStringTransform.java trunk/common-core/src/main/java/com/metamatrix/common/types/basic/SQLXMLToStringTransform.java trunk/common-core/src/main/resources/com/metamatrix/core/i18n.properties trunk/common-core/src/test/java/com/metamatrix/common/types/TestBlobValue.java trunk/common-core/src/test/java/com/metamatrix/common/types/TestClobValue.java trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/XMLSource.java trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java Log: TEIID-767 TEIID-771 simplifying the lob handling of buffermanager and merging in the memorystate logic. reviewed by RR. Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMBlob.java =================================================================== --- trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMBlob.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMBlob.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -55,7 +55,7 @@ */ public static Blob newInstance(StreamingLobChunckProducer.Factory lobChunckFactory, BlobType blob) { if (!Boolean.getBoolean(Streamable.FORCE_STREAMING)) { - Blob sourceBlob = blob.getSourceBlob(); + Blob sourceBlob = blob.getReference(); if (sourceBlob != null) { return sourceBlob; } Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java =================================================================== --- trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMClob.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -72,7 +72,7 @@ public static Clob newInstance(StreamingLobChunckProducer.Factory lobChunckFactory, ClobType clob) throws SQLException { if (!Boolean.getBoolean(Streamable.FORCE_STREAMING)) { - Clob sourceClob = clob.getSourceClob(); + Clob sourceClob = clob.getReference(); if (sourceClob != null) { return sourceClob; } Modified: trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java =================================================================== --- trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/client-jdbc/src/main/java/com/metamatrix/jdbc/MMSQLXML.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -59,7 +59,7 @@ public static SQLXML newInstance(StreamingLobChunckProducer.Factory lobChunckFactory, XMLType srcXML) throws SQLException { if (Boolean.getBoolean(Streamable.FORCE_STREAMING)) { - SQLXML sourceSQLXML = srcXML.getSourceSQLXML(); + SQLXML sourceSQLXML = srcXML.getReference(); if (sourceSQLXML != null) { return sourceSQLXML; } Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/BlobType.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/BlobType.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/BlobType.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -30,20 +30,15 @@ import java.sql.SQLException; import javax.sql.rowset.serial.SerialBlob; -import javax.sql.rowset.serial.SerialException; -import com.metamatrix.core.CorePlugin; import com.metamatrix.core.MetaMatrixRuntimeException; -import com.metamatrix.core.util.ArgCheck; /** * Represent a value of type "blob", which can be streamable from client */ -public final class BlobType implements Streamable, Blob { +public final class BlobType extends Streamable<Blob> implements Blob { - private transient Blob srcBlob; - private String streamId; - private String persistentId; + private static final long serialVersionUID = 1294191629070433450L; private long length = -1; /** @@ -54,8 +49,7 @@ } public BlobType(Blob blob) { - ArgCheck.isNotNull(blob); - this.srcBlob = blob; + super(blob); try { this.length = blob.length(); } catch (SQLException e) { @@ -63,52 +57,18 @@ } } - public Blob getSourceBlob() { - return srcBlob; - } - /** - * @see com.metamatrix.common.types.Streamable#getReferenceStreamId() - */ - public String getReferenceStreamId() { - return this.streamId; - } - - /** - * @see com.metamatrix.common.types.Streamable#setReferenceStreamId(java.lang.String) - */ - public void setReferenceStreamId(String id) { - this.streamId = id; - } - - /** - * @see com.metamatrix.common.types.Streamable#getPersistenceStreamId() - */ - public String getPersistenceStreamId() { - return persistentId; - } - - /** - * @see com.metamatrix.common.types.Streamable#setPersistenceStreamId(java.lang.String) - */ - public void setPersistenceStreamId(String id) { - this.persistentId = id; - } - - /** * @see java.sql.Blob#getBinaryStream() */ public InputStream getBinaryStream() throws SQLException { - checkReference(); - return this.srcBlob.getBinaryStream(); + return this.reference.getBinaryStream(); } /** * @see java.sql.Blob#getBytes(long, int) */ public byte[] getBytes(long pos, int length) throws SQLException { - checkReference(); - return this.srcBlob.getBytes(pos, length); + return this.reference.getBytes(pos, length); } /** @@ -121,32 +81,28 @@ } // if did not find before then do it again. - checkReference(); - return this.srcBlob.length(); + return this.reference.length(); } /** * @see java.sql.Blob#position(java.sql.Blob, long) */ public long position(Blob pattern, long start) throws SQLException { - checkReference(); - return this.srcBlob.position(pattern, start); + return this.reference.position(pattern, start); } /** * @see java.sql.Blob#position(byte[], long) */ public long position(byte[] pattern, long start) throws SQLException { - checkReference(); - return this.srcBlob.position(pattern, start); + return this.reference.position(pattern, start); } /** * @see java.sql.Blob#setBinaryStream(long) */ public OutputStream setBinaryStream(long pos) throws SQLException { - checkReference(); - return this.srcBlob.setBinaryStream(pos); + return this.reference.setBinaryStream(pos); } /** @@ -157,63 +113,23 @@ byte[] bytes, int offset, int len) throws SQLException { - checkReference(); - return this.srcBlob.setBytes(pos, bytes, offset, len); + return this.reference.setBytes(pos, bytes, offset, len); } /** * @see java.sql.Blob#setBytes(long, byte[]) */ public int setBytes(long pos, byte[] bytes) throws SQLException { - checkReference(); - return this.srcBlob.setBytes(pos, bytes); + return this.reference.setBytes(pos, bytes); } /** * @see java.sql.Blob#truncate(long) */ public void truncate(long len) throws SQLException { - checkReference(); - this.srcBlob.truncate(len); + this.reference.truncate(len); } - /** - * @see java.lang.Object#equals(java.lang.Object) - */ - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (!(o instanceof BlobType)) { - return false; - } - - BlobType other = (BlobType)o; - - if (this.srcBlob != null) { - return this.srcBlob.equals(other.srcBlob); - } - - return this.persistentId == other.persistentId - && this.streamId == other.streamId; - - } - - /** - * @see java.lang.Object#toString() - */ - public String toString() { - checkReference(); - return srcBlob.toString(); - } - - private void checkReference() { - if (this.srcBlob == null) { - throw new InvalidReferenceException(CorePlugin.Util.getString("BlobValue.InvalidReference")); //$NON-NLS-1$ - } - } - /** * Utility Method to convert blob into byte array * @param blob @@ -234,14 +150,12 @@ } //## JDBC4.0-begin ## public void free() throws SQLException { - checkReference(); - this.srcBlob.free(); + this.reference.free(); } public InputStream getBinaryStream(long pos, long length) throws SQLException { - checkReference(); - return this.srcBlob.getBinaryStream(pos, length); + return this.reference.getBinaryStream(pos, length); } //## JDBC4.0-end ## Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/ClobType.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/ClobType.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/ClobType.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -31,41 +31,27 @@ import java.sql.Clob; import java.sql.SQLException; -import javax.sql.rowset.serial.SerialBlob; import javax.sql.rowset.serial.SerialClob; -import javax.sql.rowset.serial.SerialException; -import com.metamatrix.core.CorePlugin; import com.metamatrix.core.MetaMatrixRuntimeException; /** * This is wrapper on top of a "clob" object, which implements the "java.sql.Clob" * interface. This class also implements the Streamable interface */ -public final class ClobType implements Streamable, Clob, Sequencable { +public final class ClobType extends Streamable<Clob> implements Clob, Sequencable { - private transient Clob srcClob; - private String streamId; - private String persistentId; + private static final long serialVersionUID = 2753412502127824104L; private long length = -1; /** * Can't construct */ ClobType() { - super(); } - public Clob getSourceClob() { - return this.srcClob; - } - public ClobType(Clob clob) { - if (clob == null) { - throw new IllegalArgumentException(CorePlugin.Util.getString("ClobValue.isNUll")); //$NON-NLS-1$ - } - // this will serve as the in VM reference - this.srcClob = clob; + super(clob); try { this.length = clob.length(); @@ -73,57 +59,26 @@ // ignore. } } - + /** - * @see com.metamatrix.common.types.Streamable#getReferenceStreamId() - */ - public String getReferenceStreamId() { - return this.streamId; - } - - /** - * @see com.metamatrix.common.types.Streamable#setReferenceStreamId(java.lang.String) - */ - public void setReferenceStreamId(String id) { - this.streamId = id; - } - - /** - * @see com.metamatrix.common.types.Streamable#getPersistenceStreamId() - */ - public String getPersistenceStreamId() { - return persistentId; - } - - /** - * @see com.metamatrix.common.types.Streamable#setPersistenceStreamId(java.lang.String) - */ - public void setPersistenceStreamId(String id) { - this.persistentId = id; - } - - /** * @see java.sql.Clob#getAsciiStream() */ public InputStream getAsciiStream() throws SQLException { - checkReference(); - return this.srcClob.getAsciiStream(); + return this.reference.getAsciiStream(); } /** * @see java.sql.Clob#getCharacterStream() */ public Reader getCharacterStream() throws SQLException { - checkReference(); - return this.srcClob.getCharacterStream(); + return this.reference.getCharacterStream(); } /** * @see java.sql.Clob#getSubString(long, int) */ public String getSubString(long pos, int length) throws SQLException { - checkReference(); - return this.srcClob.getSubString(pos, length); + return this.reference.getSubString(pos, length); } /** @@ -134,40 +89,35 @@ return this.length; } - checkReference(); - return this.srcClob.length(); + return this.reference.length(); } /** * @see java.sql.Clob#position(java.sql.Clob, long) */ public long position(Clob searchstr, long start) throws SQLException { - checkReference(); - return this.srcClob.position(searchstr, start); + return this.reference.position(searchstr, start); } /** * @see java.sql.Clob#position(java.lang.String, long) */ public long position(String searchstr, long start) throws SQLException { - checkReference(); - return this.srcClob.position(searchstr, start); + return this.reference.position(searchstr, start); } /** * @see java.sql.Clob#setAsciiStream(long) */ public OutputStream setAsciiStream(long pos) throws SQLException { - checkReference(); - return this.srcClob.setAsciiStream(pos); + return this.reference.setAsciiStream(pos); } /** * @see java.sql.Clob#setCharacterStream(long) */ public Writer setCharacterStream(long pos) throws SQLException { - checkReference(); - return this.srcClob.setCharacterStream(pos); + return this.reference.setCharacterStream(pos); } /** @@ -177,62 +127,23 @@ String str, int offset, int len) throws SQLException { - checkReference(); - return this.srcClob.setString(pos, str, offset, len); + return this.reference.setString(pos, str, offset, len); } /** * @see java.sql.Clob#setString(long, java.lang.String) */ public int setString(long pos, String str) throws SQLException { - checkReference(); - return this.srcClob.setString(pos, str); + return this.reference.setString(pos, str); } /** * @see java.sql.Clob#truncate(long) */ public void truncate(long len) throws SQLException { - checkReference(); - this.srcClob.truncate(len); + this.reference.truncate(len); } - /** - * @see java.lang.Object#equals(java.lang.Object) - */ - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (!(o instanceof ClobType)) { - return false; - } - - ClobType other = (ClobType)o; - - if (this.srcClob != null) { - return this.srcClob.equals(other.srcClob); - } - - return this.persistentId == other.persistentId - && this.streamId == other.streamId; - } - - /** - * @see java.lang.Object#toString() - */ - public String toString() { - checkReference(); - return srcClob.toString(); - } - - private void checkReference() { - if (this.srcClob == null) { - throw new InvalidReferenceException(CorePlugin.Util.getString("ClobValue.InvalidReference")); //$NON-NLS-1$ - } - } - /** * Utility method to convert to String * @param clob @@ -255,7 +166,6 @@ private final static int CHAR_SEQUENCE_BUFFER_SIZE = 2 << 12; public CharSequence getCharSequence() { - checkReference(); return new CharSequence() { private String buffer; @@ -299,13 +209,11 @@ } //## JDBC4.0-begin ## public void free() throws SQLException { - checkReference(); - this.srcClob.free(); + this.reference.free(); } public Reader getCharacterStream(long pos, long length) throws SQLException { - checkReference(); - return this.srcClob.getCharacterStream(pos, length); + return this.reference.getCharacterStream(pos, length); } //## JDBC4.0-end ## Deleted: trunk/common-core/src/main/java/com/metamatrix/common/types/InvalidReferenceException.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/InvalidReferenceException.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/InvalidReferenceException.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -1,34 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.types; - - -/** - * A exception class to define that a invalid reference has been accessed. - */ -public class InvalidReferenceException extends RuntimeException { - public InvalidReferenceException() {} - public InvalidReferenceException(String msg){ - super(msg); - } -} Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/Streamable.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -23,8 +23,11 @@ package com.metamatrix.common.types; import java.io.Serializable; +import java.lang.ref.PhantomReference; +import com.metamatrix.core.CorePlugin; + /** * A large value object which can be streamable in chunks of data each time * @@ -38,31 +41,71 @@ * the process worker to in case the reference object has lost its state and we * need to reinsate the object from the disk. */ -public interface Streamable extends Serializable { - static final String FORCE_STREAMING = "FORCE_STREAMING"; //$NON-NLS-1$ +public abstract class Streamable<T> implements Serializable { + public static final String FORCE_STREAMING = "FORCE_STREAMING"; //$NON-NLS-1$ public static final int STREAMING_BATCH_SIZE_IN_BYTES = 102400; // 100K + + private String referenceStreamId; + private String persistenceStreamId; + protected transient T reference; - /** - * Reference Stream ID in the server - * @return string - this is buffer managers tuple source id. - */ - String getReferenceStreamId(); + public Streamable() { + + } - /** - * Reference Stream ID in the server - * @param id this is buffer managers tuple source id. - */ - void setReferenceStreamId(String id); + public Streamable(T reference) { + if (reference == null) { + throw new IllegalArgumentException(CorePlugin.Util.getString("Streamable.isNUll")); //$NON-NLS-1$ + } + + this.reference = reference; + } - /** - * Persitence Stream ID in the server - * @return string - this is buffer managers tuple source id. - */ - String getPersistenceStreamId(); + public T getReference() { + return reference; + } - /** - * Persitence Stream ID in the server - * @param id this is buffer managers tuple source id. - */ - void setPersistenceStreamId(String id); + public void setReference(T reference) { + this.reference = reference; + } + + public String getReferenceStreamId() { + return this.referenceStreamId; + } + + public void setReferenceStreamId(String id) { + this.referenceStreamId = id; + } + + public String getPersistenceStreamId() { + return persistenceStreamId; + } + + public void setPersistenceStreamId(String id) { + this.persistenceStreamId = id; + } + + @Override + public String toString() { + return reference.toString(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Streamable<?>)) { + return false; + } + Streamable<?> other = (Streamable<?>)obj; + + if (this.reference != null) { + return this.reference.equals(other.reference); + } + + return this.persistenceStreamId == other.persistenceStreamId + && this.referenceStreamId == other.referenceStreamId; + } + } Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/XMLType.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/XMLType.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/XMLType.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -38,125 +38,57 @@ import javax.xml.transform.Result; import javax.xml.transform.Source; -import com.metamatrix.core.CorePlugin; - /** * This class represents the SQLXML object along with the Streamable interface. This is * class used everywhere in the MetaMatrix framework, but clients are restricted to use * only SQLXML interface on top of this. */ -public final class XMLType implements Streamable, SQLXML { +public final class XMLType extends Streamable<SQLXML> implements SQLXML { - private transient SQLXML srcXML; - private String referenceStreamId; - private String persistenceStreamId; + private static final long serialVersionUID = -7922647237095135723L; public XMLType(){ } - public SQLXML getSourceSQLXML() { - return srcXML; - } - public XMLType(SQLXML xml) { - if (xml == null) { - throw new IllegalArgumentException(CorePlugin.Util.getString("XMLValue.isNUll")); //$NON-NLS-1$ - } - - // this will serve as the in VM reference - this.srcXML = xml; + super(xml); } - - public String getReferenceStreamId() { - return this.referenceStreamId; - } - - public void setReferenceStreamId(String id) { - this.referenceStreamId = id; - } - - public String getPersistenceStreamId() { - return persistenceStreamId; - } - - public void setPersistenceStreamId(String id) { - this.persistenceStreamId = id; - } - + public InputStream getBinaryStream() throws SQLException { - checkReference(); - return this.srcXML.getBinaryStream(); + return this.reference.getBinaryStream(); } public Reader getCharacterStream() throws SQLException { - checkReference(); - return this.srcXML.getCharacterStream(); + return this.reference.getCharacterStream(); } public <T extends Source> T getSource(Class<T> sourceClass) throws SQLException { - checkReference(); - return this.srcXML.getSource(sourceClass); + return this.reference.getSource(sourceClass); } public String getString() throws SQLException { - checkReference(); - return this.srcXML.getString(); + return this.reference.getString(); } public OutputStream setBinaryStream() throws SQLException { - checkReference(); - return this.srcXML.setBinaryStream(); + return this.reference.setBinaryStream(); } public Writer setCharacterStream() throws SQLException { - checkReference(); - return this.srcXML.setCharacterStream(); + return this.reference.setCharacterStream(); } public void setString(String value) throws SQLException { - checkReference(); - this.srcXML.setString(value); + this.reference.setString(value); } - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (!(o instanceof XMLType)) { - return false; - } - - XMLType other = (XMLType)o; - - if (this.srcXML != null) { - return this.srcXML.equals(other.srcXML); - } - - return this.persistenceStreamId == other.persistenceStreamId - && this.referenceStreamId == other.referenceStreamId; - } - - public String toString() { - checkReference(); - return srcXML.toString(); - } - - private void checkReference() { - if (this.srcXML == null) { - throw new InvalidReferenceException(CorePlugin.Util.getString("XMLValue.InvalidReference")); //$NON-NLS-1$ - } - } - public void free() throws SQLException { - checkReference(); - this.srcXML.free(); + this.reference.free(); } public <T extends Result> T setResult(Class<T> resultClass) throws SQLException { - checkReference(); - return this.srcXML.setResult(resultClass); + return this.reference.setResult(resultClass); } } Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/basic/ClobToStringTransform.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/basic/ClobToStringTransform.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/basic/ClobToStringTransform.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -28,7 +28,6 @@ import com.metamatrix.common.types.ClobType; import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.common.types.InvalidReferenceException; import com.metamatrix.common.types.TransformationException; import com.metamatrix.core.CorePlugin; @@ -65,9 +64,7 @@ throw new TransformationException(e, CorePlugin.Util.getString("failed_convert", new Object[] {getSourceType().getName(), getTargetType().getName()})); //$NON-NLS-1$ } catch(IOException e) { throw new TransformationException(e, CorePlugin.Util.getString("failed_convert", new Object[] {getSourceType().getName(), getTargetType().getName()})); //$NON-NLS-1$ - } catch(InvalidReferenceException e) { - throw new TransformationException(e, CorePlugin.Util.getString("remote_lob_access")); //$NON-NLS-1$ - } + } } /** Modified: trunk/common-core/src/main/java/com/metamatrix/common/types/basic/SQLXMLToStringTransform.java =================================================================== --- trunk/common-core/src/main/java/com/metamatrix/common/types/basic/SQLXMLToStringTransform.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/java/com/metamatrix/common/types/basic/SQLXMLToStringTransform.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -26,7 +26,6 @@ import java.sql.SQLException; import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.common.types.InvalidReferenceException; import com.metamatrix.common.types.TransformationException; import com.metamatrix.common.types.XMLType; import com.metamatrix.core.CorePlugin; @@ -54,8 +53,6 @@ return new String(result, 0, read); } catch (SQLException e) { throw new TransformationException(e, CorePlugin.Util.getString("failed_convert", new Object[] {getSourceType().getName(), getTargetType().getName()})); //$NON-NLS-1$ - } catch(InvalidReferenceException e) { - throw new TransformationException(e, CorePlugin.Util.getString("remote_lob_access")); //$NON-NLS-1$ } catch (IOException e) { throw new TransformationException(e, CorePlugin.Util.getString("failed_convert", new Object[] {getSourceType().getName(), getTargetType().getName()})); //$NON-NLS-1$ } Modified: trunk/common-core/src/main/resources/com/metamatrix/core/i18n.properties =================================================================== --- trunk/common-core/src/main/resources/com/metamatrix/core/i18n.properties 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/main/resources/com/metamatrix/core/i18n.properties 2009-08-12 17:39:14 UTC (rev 1235) @@ -325,12 +325,8 @@ BlobImpl.Invalid_start_position=The position to begin searching, "{0}", is not valid. -BlobValue.isNUll=Blob object argument can not be null -BlobValue.InvalidReference=Blob Contents are not available, use the Streaming interface to get the contents. -ClobValue.isNUll=Clob object argument can not be null -ClobValue.InvalidReference=Clob Contents are not available, use the Streaming interface to get the contents. -XMLValue.InvalidReference=XML Contents are not available, use the Streaming interface to get the contents. -XMLValue.isNUll=SQLXML object argument can not be null +Streamable.isNUll=Streamable object argument can not be null +Streamable.InvalidReference=Streamable contents are not available, use the Streaming interface to get the contents. WorkerPool.New_thread=Created worker thread "{0}". WorkerPool.uncaughtException=Uncaught exception processing work Modified: trunk/common-core/src/test/java/com/metamatrix/common/types/TestBlobValue.java =================================================================== --- trunk/common-core/src/test/java/com/metamatrix/common/types/TestBlobValue.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/test/java/com/metamatrix/common/types/TestBlobValue.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -68,12 +68,7 @@ assertEquals(key, read.getReferenceStreamId()); // and lost the original object - try { - read.getBinaryStream(); - fail("this must thrown a reference stream exception"); //$NON-NLS-1$ - } catch (InvalidReferenceException e) { - // pass - } + assertNull(read.getReference()); saved.delete(); } Modified: trunk/common-core/src/test/java/com/metamatrix/common/types/TestClobValue.java =================================================================== --- trunk/common-core/src/test/java/com/metamatrix/common/types/TestClobValue.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/test/java/com/metamatrix/common/types/TestClobValue.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -68,12 +68,7 @@ assertEquals(key, read.getReferenceStreamId()); // and lost the original object - try { - read.getCharacterStream(); - fail("this must thrown a reference stream exception"); //$NON-NLS-1$ - } catch (InvalidReferenceException e) { - // pass - } + assertNull(read.getReference()); saved.delete(); } Modified: trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java =================================================================== --- trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/common-core/src/test/java/com/metamatrix/common/types/TestXMLValue.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -69,12 +69,7 @@ assertEquals(pkey, read.getPersistenceStreamId()); // and lost the original object - try { - read.getCharacterStream(); - fail("this must thrown a reference stream exception"); //$NON-NLS-1$ - } catch (InvalidReferenceException e) { - // pass - } + assertNull(read.getReference()); saved.delete(); } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -27,6 +27,7 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.lob.LobChunk; +import com.metamatrix.common.types.Streamable; /** * The buffer manager controls how memory is used and how data flows through @@ -268,4 +269,23 @@ * to ensure that the memory can be freed. */ void releasePinnedBatches() throws MetaMatrixComponentException; + + /** + * Return the LOB associated with the referenceId + * @param id + * @param referenceId + * @return + * @throws TupleSourceNotFoundException + * @throws MetaMatrixComponentException + */ + public Streamable<?> getStreamable(TupleSourceID id, String referenceId) + throws TupleSourceNotFoundException, MetaMatrixComponentException; + + /** + * Assign the tuplesource as the persistent stream for the streamable + * @param id + * @param s + * @throws TupleSourceNotFoundException + */ + public void setPersistentTupleSource(TupleSourceID id, Streamable<? extends Object> s) throws TupleSourceNotFoundException; } Copied: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java (from rev 1231, trunk/engine/src/main/java/com/metamatrix/common/lob/BufferManagerLobChunkStream.java) =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -0,0 +1,67 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer; + + +import java.io.IOException; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.common.lob.LobChunk; +import com.metamatrix.common.lob.LobChunkProducer; +import com.metamatrix.common.log.LogManager; +import com.metamatrix.dqp.DQPPlugin; +import com.metamatrix.dqp.util.LogConstants; + +public class BufferManagerLobChunkStream implements LobChunkProducer { + private TupleSourceID sourceId; + private BufferManager bufferMgr; + private int position = 0; + + public BufferManagerLobChunkStream(String persitentId, BufferManager bufferMgr) { + this.sourceId = new TupleSourceID(persitentId); + this.bufferMgr = bufferMgr; + } + + public LobChunk getNextChunk() throws IOException { + try { + this.position++; + return bufferMgr.getStreamablePart(sourceId, position); + } catch (TupleSourceNotFoundException e) { + String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", new Object[] {sourceId}); //$NON-NLS-1$ + LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); + throw new IOException(msg); + } catch (MetaMatrixComponentException e) { + String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.error_processing", new Object[] {sourceId}); //$NON-NLS-1$ + LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); + throw new IOException(msg); + } + } + + /** + * @see com.metamatrix.common.lob.LobChunkProducer#close() + */ + public void close() throws IOException { + // we could remove the buffer tuple here but, this is just a stream, so we need to delete + // that when we close th eplan. + } +} \ No newline at end of file Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/LobTupleBatch.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/LobTupleBatch.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/LobTupleBatch.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -1,40 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -import java.util.List; - - - -/** - * A marker class file holding the lob based data in a separate batch holder. - */ -public class LobTupleBatch extends TupleBatch { - - public LobTupleBatch() { - } - - public LobTupleBatch(int beginRow, List listOfTupleLists) { - super(beginRow, listOfTupleLists); - } -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -25,7 +25,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Properties; @@ -39,7 +41,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.LobTupleBatch; import com.metamatrix.common.buffer.MemoryNotAvailableException; import com.metamatrix.common.buffer.StorageManager; import com.metamatrix.common.buffer.TupleBatch; @@ -50,23 +51,28 @@ import com.metamatrix.common.types.Streamable; import com.metamatrix.core.log.MessageLevel; import com.metamatrix.core.util.Assertion; +import com.metamatrix.dqp.DQPPlugin; import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.query.execution.QueryExecPlugin; /** * Default implementation of BufferManager. This buffer manager implementation * assumes the usage of a StorageManager of type memory and optionally (preferred) - * an additional StorageManager of type FILE or DISK. If no persistent manager - * is specified, everything managed by this BufferManager is assumed to fit in - * memory. This can be useful for testing or for small uses. - * - * Lots of state is cached in memory. The tupleSourceMap contains a map of - * TupleSourceID --> TupleSourceInfo. Everything about a particular tuple - * source is stored there. The memoryState contains everything pertaining to - * memory management. The config contains all config info. + * an additional StorageManager of type FILE. */ public class BufferManagerImpl implements BufferManager { - + + //memory availability when reserveMemory() is called + static final int MEMORY_AVAILABLE = 1; + static final int MEMORY_EXCEED_MAX = 2; //exceed buffer manager max memory + static final int MEMORY_EXCEED_SESSION_MAX = 3; //exceed session max memory + + private static ThreadLocal<Set<ManagedBatch>> PINNED_BY_THREAD = new ThreadLocal<Set<ManagedBatch>>() { + protected Set<ManagedBatch> initialValue() { + return new HashSet<ManagedBatch>(); + }; + }; + // Initialized stuff private String lookup; private BufferConfig config; @@ -76,15 +82,18 @@ // groupName (String) -> TupleGroupInfo map private Map<String, TupleGroupInfo> groupInfos = new HashMap<String, TupleGroupInfo>(); - // Storage managers + // Storage manager private StorageManager diskMgr; // ID creator private AtomicLong currentTuple = new AtomicLong(0); - // Memory management - private MemoryState memoryState; - + // Keep track of how memory usage + private volatile long memoryUsed = 0; + + // Track the currently unpinned stuff in a sorted set + private Set<ManagedBatch> unpinned = Collections.synchronizedSet(new LinkedHashSet<ManagedBatch>()); + // Trigger to handle management and stats logging private Timer timer; @@ -94,6 +103,7 @@ private AtomicInteger pinnedFromDisk = new AtomicInteger(0); private AtomicInteger cleanings = new AtomicInteger(0); private AtomicLong totalCleaned = new AtomicLong(0); + private AtomicInteger pinned = new AtomicInteger(); /** * See {@link com.metamatrix.common.buffer.BufferManagerPropertyNames} for a @@ -108,15 +118,13 @@ // Set up config based on properties this.config = new BufferConfig(properties); - // Set up memory state object - this.memoryState = new MemoryState(config); - // Set up alarms based on config if(this.config.getManagementInterval() > 0) { TimerTask mgmtTask = new TimerTask() { public void run() { - clean(0); + clean(0, null); } + }; getTimer().schedule(mgmtTask, 0, this.config.getManagementInterval()); } @@ -156,7 +164,8 @@ BufferStats stats = new BufferStats(); // Get memory info - this.memoryState.fillStats(stats); + stats.memoryUsed = this.memoryUsed; + stats.memoryFree = config.getTotalAvailableMemory() - memoryUsed; // Get picture of what's happening Set<TupleSourceID> copyKeys = tupleSourceMap.keySet(); @@ -245,11 +254,8 @@ TupleSourceID newID = new TupleSourceID(String.valueOf(this.currentTuple.getAndIncrement()), this.lookup); TupleGroupInfo tupleGroupInfo = getGroupInfo(groupName); TupleSourceInfo info = new TupleSourceInfo(newID, schema, types, tupleGroupInfo, tupleSourceType); - synchronized (tupleGroupInfo) { - tupleGroupInfo.getTupleSourceIDs().add(newID); - } + tupleGroupInfo.getTupleSourceIDs().add(newID); tupleSourceMap.put(newID, info); - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Creating TupleSource:", newID, "of type "+tupleSourceType}); //$NON-NLS-1$ //$NON-NLS-2$ } @@ -289,31 +295,25 @@ ManagedBatch batch = iter.next(); switch(batch.getLocation()) { case ManagedBatch.UNPINNED: - memoryState.removeUnpinned(batch); - memoryState.releaseMemory(batch.getSize(), info.getGroupInfo()); + this.unpinned.remove(batch); + releaseMemory(batch.getSize(), info.getGroupInfo()); break; case ManagedBatch.PINNED: - memoryState.removePinned(info.getTupleSourceID(), batch.getBeginRow()); - memoryState.releaseMemory(batch.getSize(), info.getGroupInfo()); + PINNED_BY_THREAD.get().remove(batch); + this.pinned.getAndDecrement(); + releaseMemory(batch.getSize(), info.getGroupInfo()); break; } } } } TupleGroupInfo tupleGroupInfo = info.getGroupInfo(); - synchronized (tupleGroupInfo) { - tupleGroupInfo.getTupleSourceIDs().remove(tupleSourceID); - } + tupleGroupInfo.getTupleSourceIDs().remove(tupleSourceID); // Remove disk storage if (this.diskMgr != null){ this.diskMgr.removeBatches(tupleSourceID); } - - // remove any dependent tuple sources on this tuple source - // lob frame work uses the parent tuple's name as group name to - // tie it back to the original source. - removeTupleSources(tupleSourceID.getStringID()); } /** @@ -338,28 +338,25 @@ return; } List<TupleSourceID> tupleSourceIDs = null; - synchronized (tupleGroupInfo) { + synchronized (tupleGroupInfo.getTupleSourceIDs()) { tupleSourceIDs = new ArrayList<TupleSourceID>(tupleGroupInfo.getTupleSourceIDs()); } - // Remove them - if(tupleSourceIDs.size() > 0) { - MetaMatrixComponentException ex = null; + MetaMatrixComponentException ex = null; - for (TupleSourceID tsID : tupleSourceIDs) { - try { - this.removeTupleSource(tsID); - } catch(TupleSourceNotFoundException e) { - // ignore and go on - } catch(MetaMatrixComponentException e) { - if(ex == null) { - ex = e; - } - } - } + for (TupleSourceID tsID : tupleSourceIDs) { + try { + this.removeTupleSource(tsID); + } catch(TupleSourceNotFoundException e) { + // ignore and go on + } catch(MetaMatrixComponentException e) { + if(ex == null) { + ex = e; + } + } + } - if(ex != null) { - throw ex; - } + if(ex != null) { + throw ex; } } @@ -466,7 +463,7 @@ // if there are lobs in source then we need to keep manage then // in a separate tuple sources. if (info.lobsInSource()) { - createTupleSourcesForLobs(tupleSourceID, tupleBatch); + correctLobReferences(info, tupleBatch); } // Determine where to store @@ -474,13 +471,13 @@ tupleBatch.setSize(bytes); short location = ManagedBatch.PERSISTENT; - if(memoryState.reserveMemory(bytes, info.getGroupInfo()) == MemoryState.MEMORY_AVAILABLE) { + if(reserveMemory(bytes, info.getGroupInfo()) == BufferManagerImpl.MEMORY_AVAILABLE) { location = ManagedBatch.UNPINNED; } synchronized(info) { if(info.isRemoved()) { - memoryState.releaseMemory(bytes, info.getGroupInfo()); + releaseMemory(bytes, info.getGroupInfo()); throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$ } @@ -498,14 +495,14 @@ } catch(MetaMatrixComponentException e) { // If we were storing to memory, clean up memory we reserved if(location != ManagedBatch.PERSISTENT) { - memoryState.releaseMemory(bytes, info.getGroupInfo()); + releaseMemory(bytes, info.getGroupInfo()); } throw e; } // Add to memory state if in memory if(location == ManagedBatch.UNPINNED) { - this.memoryState.addUnpinned(managedBatch); + this.unpinned.add(managedBatch); } // Update info with new rows @@ -533,18 +530,39 @@ this.pinRequests.incrementAndGet(); - TupleBatch memoryBatch = null; + TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); + int memoryAvailability = BufferManagerImpl.MEMORY_AVAILABLE; + + ManagedBatch mbatch = null; + synchronized (info) { + mbatch = info.getBatch(beginRow); + } + if(mbatch == null) { + return new TupleBatch(beginRow, Collections.EMPTY_LIST); + } + int endRow = 0; - int count = 0; int pass = 0; + //if the client request previous batch, the end row + //is smaller than the begin row + if(beginRow > maxEndRow) { + endRow = Math.min(beginRow, mbatch.getEndRow()); + beginRow = Math.max(maxEndRow, mbatch.getBeginRow()); + }else { + endRow = Math.min(maxEndRow, mbatch.getEndRow()); + } + int count = endRow - beginRow + 1; + if (count == 0) { + return new TupleBatch(beginRow, Collections.EMPTY_LIST); + } + long memoryRequiredByBatch = mbatch.getSize(); - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - long memoryRequiredByBatch = 0; - int memoryAvailability = MemoryState.MEMORY_AVAILABLE; + TupleBatch memoryBatch = null; + while(pass < 2) { if(pass == 1) { - if(memoryAvailability == MemoryState.MEMORY_EXCEED_MAX) { - clean(memoryRequiredByBatch); + if(memoryAvailability == BufferManagerImpl.MEMORY_EXCEED_MAX) { + clean(memoryRequiredByBatch, null); }else { //exceed session limit clean(memoryRequiredByBatch, info.getGroupInfo()); @@ -552,28 +570,23 @@ } synchronized(info) { - ManagedBatch mbatch = info.getBatch(beginRow); - if(mbatch == null) { - return new TupleBatch(beginRow, Collections.EMPTY_LIST); - - } else if(mbatch.getLocation() == ManagedBatch.PINNED) { + if(mbatch.getLocation() == ManagedBatch.PINNED) { // Load batch from memory - already pinned memoryBatch = mbatch.getBatch(); } else if(mbatch.getLocation() == ManagedBatch.UNPINNED) { // Already in memory - just move from unpinned to pinned - mbatch.setLocation(ManagedBatch.PINNED); - this.memoryState.removeUnpinned(mbatch); - this.memoryState.addPinned(mbatch); + this.unpinned.remove(mbatch); + pin(mbatch); // Load batch from memory memoryBatch = mbatch.getBatch(); - } else if(mbatch.getLocation() == ManagedBatch.PERSISTENT) { + } else { memoryRequiredByBatch = mbatch.getSize(); // Try to reserve some memory - if((memoryAvailability = memoryState.reserveMemory(memoryRequiredByBatch, info.getGroupInfo())) != MemoryState.MEMORY_AVAILABLE) { + if((memoryAvailability = reserveMemory(memoryRequiredByBatch, info.getGroupInfo())) != BufferManagerImpl.MEMORY_AVAILABLE) { if(pass == 0) { // Break and try to clean - it is important to break out of the synchronized block // here so that the clean does not cause a deadlock on this TupleSourceInfo @@ -595,22 +608,11 @@ memoryBatch = diskMgr.getBatch(tupleSourceID, internalBeginRow, info.getTypes()); mbatch.setBatch(memoryBatch); - mbatch.setLocation(ManagedBatch.PINNED); - this.memoryState.addPinned(mbatch); + if (info.lobsInSource()) { + correctLobReferences(info, memoryBatch); + } + pin(mbatch); } - - //if the client request previous batch, the end row - //is smaller than the begin row - if(beginRow > maxEndRow) { - endRow = Math.min(beginRow, memoryBatch.getEndRow()); - beginRow = Math.max(maxEndRow, memoryBatch.getBeginRow()); - }else { - endRow = Math.min(maxEndRow, memoryBatch.getEndRow()); - } - count = endRow - beginRow + 1; - if(count > 0) { - mbatch.pin(); - } } break; @@ -618,7 +620,7 @@ // Batch should now be pinned in memory, so grab it and build a correctly // sized batch to return - if(memoryBatch.getRowCount() == 0 || count == 0 || (beginRow == memoryBatch.getBeginRow() && count == memoryBatch.getRowCount())) { + if(beginRow == memoryBatch.getBeginRow() && count == memoryBatch.getRowCount()) { return memoryBatch; } @@ -628,6 +630,15 @@ System.arraycopy(memoryRows, firstOffset, rows, 0, count); return new TupleBatch(beginRow, rows); } + + private void pin(ManagedBatch mbatch) { + mbatch.setLocation(ManagedBatch.PINNED); + PINNED_BY_THREAD.get().add(mbatch); + if (!mbatch.hasPinnedRows()) { + pinned.getAndIncrement(); + } + mbatch.pin(); + } /** * Unpin a tuple source batch. @@ -655,8 +666,9 @@ // Determine whether batch itself should be unpinned if(! mbatch.hasPinnedRows()) { mbatch.setLocation(ManagedBatch.UNPINNED); - memoryState.removePinned(tupleSourceID, mbatch.getBeginRow()); - memoryState.addUnpinned(mbatch); + PINNED_BY_THREAD.get().remove(mbatch); + this.unpinned.add(mbatch); + pinned.getAndDecrement(); } } } @@ -698,83 +710,71 @@ } return info; } - + /** - * Clean the memory state, using LRU. This can be done either via the background - * cleaning thread or actively if someone wants memory and none is free. + * This can be done actively if someone wants memory and none is free. */ - protected void clean(long memoryRequired) { - // Defect 14573 - this method needs to know how much memory is required, so that (even if we're not past the active memory - // threshold) if the memory available is less than the memory required, we should clean up unpinned batches. + protected void clean(long memoryRequired, TupleGroupInfo targetGroupInfo) { + cleanLobTupleSource(); + + long released = 0; + long targetLevel = config.getActiveMemoryLevel(); long totalMemory = config.getTotalAvailableMemory(); - long released = 0; - - Iterator<ManagedBatch> unpinnedIter = this.memoryState.getAllUnpinned(); - while(unpinnedIter.hasNext() && // If there are unpinned batches in memory, AND - // Defect 14573 - if we require more than what's available, then cleanup regardless of the threshold - (memoryRequired > totalMemory - memoryState.getMemoryUsed() || // if the memory needed is more than what's available, or - memoryState.getMemoryUsed() > targetLevel)){ // if we've crossed the active memory threshold, then cleanup - - ManagedBatch batch = unpinnedIter.next(); + + boolean generalCleaningDone = false; + List<ManagedBatch> toClean = null; + synchronized (unpinned) { + //TODO: re-implement without having to scan and compete + toClean = new ArrayList<ManagedBatch>(unpinned); + } + for (ManagedBatch batch : toClean) { TupleSourceID tsID = batch.getTupleSourceID(); - - released += releaseMemory(batch, tsID); - } - - if(released > 0) { - this.cleanings.incrementAndGet(); - this.totalCleaned.addAndGet(released); - } - - } - - /** - * Over memory limit for this session. Clean the memory for this session. - * Clean the memory state, using LRU. This can be done actively if someone wants memory and none is free. - */ - protected void clean(long memoryRequired, TupleGroupInfo targetGroupInfo) throws TupleSourceNotFoundException{ - boolean cleanForSessionSucceeded = false; - long released = 0; - - Iterator<ManagedBatch> unpinnedIter = this.memoryState.getAllUnpinned(); - while(unpinnedIter.hasNext()) { - ManagedBatch batch = unpinnedIter.next(); - TupleSourceID tsID = batch.getTupleSourceID(); - TupleSourceInfo tsInfo = getTupleSourceInfo(tsID, false); + TupleSourceInfo tsInfo = this.tupleSourceMap.get(tsID); if(tsInfo == null) { //may be removed by another thread continue; } - if(!tsInfo.getGroupInfo().equals(targetGroupInfo)) { - //continue if they are not the same tuple group - continue; + + long currentMemoryUsed = memoryUsed; + if (!generalCleaningDone && (memoryRequired <= totalMemory - currentMemoryUsed && // if the memory needed is more than what's available, or + currentMemoryUsed <= targetLevel)) { // if we've crossed the active memory threshold, then cleanup + generalCleaningDone = true; } + if (generalCleaningDone) { + if (targetGroupInfo == null) { + break; + } + if (targetGroupInfo == tsInfo.getGroupInfo()) { + //if the memory needed is more than what is available for the session, then cleanup. Otherwise, break the loop. + if(memoryRequired <= config.getMaxAvailableSession() - targetGroupInfo.getGroupMemoryUsed()) { + break; + } + } + } - long groupMemoryUsed = memoryState.getGroupMemoryUsed(targetGroupInfo); - //if the memory needed is more than what is available for the session, then cleanup. Otherwise, break the loop. - if(memoryRequired <= config.getMaxAvailableSession() - groupMemoryUsed) { - cleanForSessionSucceeded = true; - break; - } - - released += releaseMemory(batch, tsID); + released += releaseMemory(batch, tsInfo); } if(released > 0) { this.cleanings.incrementAndGet(); this.totalCleaned.addAndGet(released); } - - if(!cleanForSessionSucceeded) { - //if we cannot clean enough memory for this session, it fails - return; + } + + //TODO: run asynch + private void cleanLobTupleSource() { + String tupleSourceId = TupleSourceInfo.getStaleLobTupleSource(); + if (tupleSourceId != null) { + try { + removeTupleSource(new TupleSourceID(tupleSourceId)); + } catch (TupleSourceNotFoundException e) { + } catch (MetaMatrixComponentException e) { + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, e, "Exception removing stale lob tuple source"); //$NON-NLS-1$ + } } + } - //make sure it is not over the buffer manager memory limit - clean(memoryRequired); - } - /** * Release the memory for the given unpinned batch. * @param batch Batch to be released from memory @@ -782,48 +782,32 @@ * @return The size of memory released in bytes * @since 4.3 */ - private long releaseMemory(ManagedBatch batch, TupleSourceID tsID) { + private long releaseMemory(ManagedBatch batch, TupleSourceInfo info) { // Find info and lock on it - try { - TupleSourceInfo info = getTupleSourceInfo(tsID, false); - if(info == null) { + synchronized(info) { + if(info.isRemoved() || batch.getLocation() != ManagedBatch.UNPINNED) { return 0; } - synchronized(info) { - if(info.isRemoved()) { - return 0; - } + // This batch is still unpinned - move to persistent storage + TupleBatch dataBatch = batch.getBatch(); - // Re-get the batch and check that it still exists and is unpinned - batch = info.getBatch(batch.getBeginRow()); - if(batch == null || batch.getLocation() != ManagedBatch.UNPINNED) { - return 0; - } + try { + diskMgr.addBatch(info.getTupleSourceID(), dataBatch, info.getTypes()); + } catch(MetaMatrixComponentException e) { + // Can't move + return 0; + } - // This batch is still unpinned - move to persistent storage - TupleBatch dataBatch = batch.getBatch(); + batch.setBatch(null); - try { - diskMgr.addBatch(tsID, dataBatch, info.getTypes()); - } catch(MetaMatrixComponentException e) { - // Can't move - return 0; - } + // Update memory + batch.setLocation(ManagedBatch.PERSISTENT); + this.unpinned.remove(batch); + releaseMemory(batch.getSize(), info.getGroupInfo()); - batch.setBatch(null); - - // Update memory - batch.setLocation(ManagedBatch.PERSISTENT); - memoryState.removeUnpinned(batch); - memoryState.releaseMemory(batch.getSize(), info.getGroupInfo()); - - return batch.getSize(); - } - } catch(TupleSourceNotFoundException e) { - // ignore, go to next batch - return 0; - } + return batch.getSize(); + } } /** @@ -855,114 +839,49 @@ } /** - * If a tuple batch is being added with Lobs, then maintain the LOB - * objects in a separate TupleSource than from the original, so that - * the original can only serilize the id, but the otherone can serialize - * the contents. + * If a tuple batch is being added with Lobs, then references to + * the lobs will be held on the {@link TupleSourceInfo} * @param batch */ - private void createTupleSourcesForLobs(TupleSourceID parentId, TupleBatch batch) - throws MetaMatrixComponentException, TupleSourceNotFoundException { - - TupleSourceInfo info = getTupleSourceInfo(parentId, false); + @SuppressWarnings("unchecked") + private void correctLobReferences(TupleSourceInfo info, TupleBatch batch) { List parentSchema = info.getTupleSchema(); List[] rows = batch.getAllTuples(); - + int columns = parentSchema.size(); // walk through the results and find all the lobs for (int row = 0; row < rows.length; row++) { - - int col = 0; - for (Iterator i = rows[row].iterator(); i.hasNext();) { - Object anObj = i.next(); + for (int col = 0; col < columns; col++) { + Object anObj = rows[row].get(col); - if (anObj instanceof Streamable) { - // once lob is found check to see if this has already been assigned - // a streming id or not; if one is not assigned create one and assign it - // to the lob; if one is already assigned just return; - // this will prohibit calling lob on itself into this routine. - Streamable lob = (Streamable)anObj; - - if (lob.getReferenceStreamId() == null || lobIsNotKnownInTupleSourceMap( lob, parentId) ) { - List schema = new ArrayList(); - schema.add(parentSchema.get(col)); - - TupleSourceID id = createTupleSource(schema, new String[] {info.getTypes()[col]}, parentId.getStringID(), TupleSourceType.PROCESSOR); - lob.setReferenceStreamId(id.getStringID()); - - List results = new ArrayList(); - results.add(lob); - - List listOfRows = new ArrayList(); - listOfRows.add(results); - - // these batches are wrapped in a special marker batch tag - // which are saved from forcing them to disk. - LobTupleBatch separateBatch = new LobTupleBatch(1, listOfRows); - separateBatch.setTerminationFlag(true); - - // now save this as separate tuple source. - addTupleBatch(id, separateBatch); - } else { - // this means the XML object being moved from one tuple to another tuple - // i.e. one plan to another plan. So update the group info. - - // First update the reference tuple source. - if (!lob.getReferenceStreamId().equals(parentId.getStringID())) { - TupleGroupInfo groupInfo = getGroupInfo(parentId.getStringID()); - TupleSourceID id = new TupleSourceID(lob.getReferenceStreamId()); - TupleSourceInfo lobInfo = getTupleSourceInfo(id, false); - reassignGroup(groupInfo, lobInfo); - - // if the lob moving parent has a assosiated persistent - // tuple source, then move that one to same parent too. - if (lob.getPersistenceStreamId() != null) { - id = new TupleSourceID(lob.getPersistenceStreamId()); - lobInfo = getTupleSourceInfo(id, false); - reassignGroup(groupInfo, lobInfo); - } - } - } + if (!(anObj instanceof Streamable<?>)) { + continue; } - col++; + Streamable lob = (Streamable)anObj; + info.addLobReference(lob); + if (lob.getReference() == null) { + lob.setReference(info.getLobReference(lob.getReferenceStreamId()).getReference()); + } } } } - - private void reassignGroup(TupleGroupInfo groupInfo, TupleSourceInfo lobInfo) { - TupleGroupInfo tupleGroupInfo = lobInfo.getGroupInfo(); - synchronized (tupleGroupInfo) { - tupleGroupInfo.getTupleSourceIDs().remove(lobInfo.getTupleSourceID()); - } - lobInfo.setGroupInfo(groupInfo); - synchronized (groupInfo) { - groupInfo.getTupleSourceIDs().add(lobInfo.getTupleSourceID()); - } - } - private boolean lobIsNotKnownInTupleSourceMap( Streamable lob, TupleSourceID parentId) throws TupleSourceNotFoundException { - /* - * The need for this defensive feature arises because there are multiple uses of the TupleSourceMap which - * are somewhat inconsistent with one another. In the case of LOBs we use the parent/child group feature - * of tuplesources to associate a parent tuplesource containing metadata about the LOB with a second - * tuplesource that contains the LOB. When such a group is no longer needed (for example, see SubqueryProcessorUtility.close()), - * removing the child tupleSources has the unfortunate side effect of leaving the actual LOBs with references to - * tuplesources that no longer exist, and are therefore no longer in the tupleSourceMap. - * - * This test ensures that such orphaned LOBs will be treated correctly (TEIID-54). - * - */ - if (!lob.getReferenceStreamId().equals(parentId.getStringID())) { - TupleSourceID id = new TupleSourceID(lob.getReferenceStreamId()); - TupleSourceInfo lobInfo = getTupleSourceInfo(id, false); - - if ( lobInfo == null ) { - return true; // is not known - } - return false; // is known - } - return false; // don't care if known + @Override + public Streamable<?> getStreamable(TupleSourceID id, String referenceId) throws TupleSourceNotFoundException, MetaMatrixComponentException { + TupleSourceInfo tsInfo = getTupleSourceInfo(id, true); + Streamable<?> s = tsInfo.getLobReference(referenceId); + if (s == null) { + throw new MetaMatrixComponentException(DQPPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ + } + return s; } + @Override + public void setPersistentTupleSource(TupleSourceID id, Streamable<?> s) throws TupleSourceNotFoundException { + cleanLobTupleSource(); + TupleSourceInfo tsInfo = getTupleSourceInfo(id, true); + s.setPersistenceStreamId(id.getStringID()); + tsInfo.setContainingLobReference(s); + } /** * @see com.metamatrix.common.buffer.BufferManager#addStreamablePart(com.metamatrix.common.buffer.TupleSourceID, com.metamatrix.common.lob.LobChunk, int) @@ -976,7 +895,7 @@ synchronized(info) { - List data = new ArrayList(); + List<LobChunk> data = new ArrayList<LobChunk>(); data.add(streamChunk); TupleBatch batch = new TupleBatch(beginRow, new List[] {data}); this.diskMgr.addBatch(tupleSourceID, batch, info.getTypes()); @@ -1008,44 +927,62 @@ * @see com.metamatrix.common.buffer.BufferManager#releasePinnedBatches() */ public void releasePinnedBatches() throws MetaMatrixComponentException { - Map<TupleSourceID, Map<Integer, ManagedBatch>> threadPinned = memoryState.getPinnedByCurrentThread(); - if (threadPinned == null) { - return; - } - for (Iterator<Map.Entry<TupleSourceID, Map<Integer, ManagedBatch>>> i = threadPinned.entrySet().iterator(); i.hasNext();) { - Map.Entry<TupleSourceID, Map<Integer, ManagedBatch>> entry = i.next(); - i.remove(); - TupleSourceID tsid = entry.getKey(); - Map<Integer, ManagedBatch> pinnedBatches = entry.getValue(); - try { - for (Iterator<ManagedBatch> j = pinnedBatches.values().iterator(); j.hasNext();) { - ManagedBatch batch = j.next(); - - //TODO: add trace logging about the batch that is being unpinned - unpinTupleBatch(tsid, batch.getBeginRow(), batch.getEndRow()); - } - } catch (TupleSourceNotFoundException err) { - continue; - } - } + MetaMatrixComponentException e = null; + List<ManagedBatch> pinnedByThread = new ArrayList<ManagedBatch>(PINNED_BY_THREAD.get()); + for (ManagedBatch managedBatch : pinnedByThread) { + try { + //TODO: add trace logging about the batch that is being unpinned + unpinTupleBatch(managedBatch.getTupleSourceID(), managedBatch.getBeginRow(), managedBatch.getEndRow()); + } catch (TupleSourceNotFoundException err) { + } catch (MetaMatrixComponentException err) { + e = err; + } + } + if (e != null) { + throw e; + } } /** - * for testing purposes + * Check for whether the specified amount of memory can be reserved, + * and if so reserve it. This is done in the same method so that the + * memory is not taken away by a different thread between checking and + * reserving - standard "test and set" behavior. + * @param bytes Bytes requested + * @return One of MEMORY_AVAILABLE, MEMORY_EXCEED_MAX, or MEMORY_EXCEED_SESSION_MAX */ - public int getPinnedCount() { - Map<TupleSourceID, Map<Integer, ManagedBatch>> pinned = memoryState.getAllPinned(); - - int count = 0; - - if (pinned == null) { - return count; + private synchronized int reserveMemory(long bytes, TupleGroupInfo groupInfo) { + //check session limit first + long sessionMax = config.getMaxAvailableSession(); + if(sessionMax - groupInfo.getGroupMemoryUsed() < bytes) { + return BufferManagerImpl.MEMORY_EXCEED_SESSION_MAX; } - for (Iterator<Map<Integer, ManagedBatch>> i = pinned.values().iterator(); i.hasNext();) { - count += i.next().size(); + //then check the total memory limit + long max = config.getTotalAvailableMemory(); + if(max - memoryUsed < bytes) { + return BufferManagerImpl.MEMORY_EXCEED_MAX; } - return count; + groupInfo.reserveMemory(bytes); + memoryUsed += bytes; + + return BufferManagerImpl.MEMORY_AVAILABLE; } + + /** + * Release memory + * @param bytes Bytes to release + */ + private synchronized void releaseMemory(long bytes, TupleGroupInfo groupInfo) { + groupInfo.releaseMemory(bytes); + memoryUsed -= bytes; + } + + /** + * for testing purposes + */ + public int getPinnedCount() { + return pinned.get(); + } } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -54,7 +54,7 @@ public long totalCleaned; // Pinned batch details - public List pinnedManagedBatches = new LinkedList(); + public List<ManagedBatch> pinnedManagedBatches = new LinkedList<ManagedBatch>(); /** * Constructor for BufferStats. @@ -90,7 +90,7 @@ LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " avgCleaned = " + avgCleaned); //$NON-NLS-1$ if ( LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE) ) { - HashMap stackTraces = new HashMap(); + HashMap<List<String>, Integer> stackTraces = new HashMap<List<String>, Integer>(); if ( pinnedManagedBatches.isEmpty() ) { return; @@ -102,12 +102,10 @@ int stackNumber = 1; // pinned batch details - Iterator it = pinnedManagedBatches.iterator(); - while ( it.hasNext() ) { - ManagedBatch batch = (ManagedBatch)it.next(); + for (ManagedBatch batch : pinnedManagedBatches) { LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " TupleSourceID: " + batch.getTupleSourceID() + " Begin: " + batch.getBeginRow() + " End: " + batch.getEndRow()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - Integer stackKey = (Integer)stackTraces.get(batch.getCallStack()); + Integer stackKey = stackTraces.get(batch.getCallStack()); boolean isFirst = false; @@ -119,7 +117,7 @@ LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " Pinned at: " + batch.getCallStackTimeStamp() + " by call# " + stackKey); //$NON-NLS-1$ //$NON-NLS-2$ if (isFirst) { - for (Iterator j = batch.getCallStack().iterator(); j.hasNext();) { + for (Iterator<String> j = batch.getCallStack().iterator(); j.hasNext();) { LogManager.logTrace( LogConstants.CTX_BUFFER_MGR, " " + j.next() ); //$NON-NLS-1$ } } Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryState.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryState.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryState.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -1,260 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.log.LogManager; -import com.metamatrix.core.log.MessageLevel; -import com.metamatrix.dqp.util.LogConstants; - -/** - * This class represents the memory state of the BufferManagerImpl. The - * critical thing to know is what batches are in memory but not being - * used and what batches in memory but being used. The access patterns - * for these two types of information are very different, so they are stored - * in very different data structures. - * - * The unpinned batches are stored in a linked list, ordered by a least - * recently used timestamp. This is optimized for cleanup. It's not very good - * for finding stuff but fortunately, we don't need to most of the time. - * - * The pinned batches are stored in a Map, keyed by TupleSourceID. The value - * is another map, keyed by beginRow, with a ManagedBatch as the value. This - * is really good for finding batches quickly, which is exactly what we need - * to do with pinned batches. - * - * All methods on this class are synchronized to preserve state. - */ -class MemoryState { - - private static ThreadLocal<Map<TupleSourceID, Map<Integer, ManagedBatch>>> PINNED_BY_THREAD = new ThreadLocal<Map<TupleSourceID, Map<Integer, ManagedBatch>>>(); - - //memory availability when reserveMemory() is called - static final int MEMORY_AVAILABLE = 1; - static final int MEMORY_EXCEED_MAX = 2; //exceed buffer manager max memory - static final int MEMORY_EXCEED_SESSION_MAX = 3; //exceed session max memory - - // Configuration, used to get available memory info - private BufferConfig config; - - // Keep track of how memory we are using - private volatile long memoryUsed = 0; - - // Track the currently pinned stuff by TupleSourceID for easy lookup - private Map<TupleSourceID, Map<Integer, ManagedBatch>> pinned = new HashMap<TupleSourceID, Map<Integer, ManagedBatch>>(); - - // Track the currently unpinned stuff in a sorted list, sorted by access time - private Set<ManagedBatch> unpinned = Collections.synchronizedSet(new LinkedHashSet<ManagedBatch>()); - - /** - * Constructor for MemoryState, based on config. - * @param config Configuration - */ - public MemoryState(BufferConfig config) { - this.config = config; - } - - /** - * Fill the stats object with stats about memory - * @param stats Stats info to be filled - */ - public void fillStats(BufferStats stats) { - stats.memoryUsed = this.memoryUsed; - stats.memoryFree = config.getTotalAvailableMemory() - memoryUsed; - } - - /** - * Get the amount of memory currently being used in bytes - * @return Used memory, in bytes - */ - public long getMemoryUsed() { - return this.memoryUsed; - } - - /** - * Check for whether the specified amount of memory can be reserved, - * and if so reserve it. This is done in the same method so that the - * memory is not taken away by a different thread between checking and - * reserving - standard "test and set" behavior. - * @param bytes Bytes requested - * @return One of MEMORY_AVAILABLE, MEMORY_EXCEED_MAX, or MEMORY_EXCEED_SESSION_MAX - */ - public synchronized int reserveMemory(long bytes, TupleGroupInfo groupInfo) { - //check session limit first - long sessionMax = config.getMaxAvailableSession(); - if(sessionMax - groupInfo.getGroupMemoryUsed() < bytes) { - return MEMORY_EXCEED_SESSION_MAX; - } - - //then check the total memory limit - long max = config.getTotalAvailableMemory(); - if(max - memoryUsed < bytes) { - return MEMORY_EXCEED_MAX; - } - - /* NOTE1 - * Since the groupInfo call is being made in the synchronized block for the entire buffer, - * groupInfo doesn't need additional locking. - */ - groupInfo.reserveMemory(bytes); - memoryUsed += bytes; - - return MEMORY_AVAILABLE; - } - - /** - * Release memory - * @param bytes Bytes to release - */ - public synchronized void releaseMemory(long bytes, TupleGroupInfo groupInfo) { - // see NOTE1 - groupInfo.releaseMemory(bytes); - memoryUsed -= bytes; - } - - /** - * Get the amount of memory currently being used for the specified group in bytes - * @param groupInfo TupleGroupInfo - * @return Used memory, in bytes - */ - public synchronized long getGroupMemoryUsed(TupleGroupInfo groupInfo) { - // see NOTE1 - return groupInfo.getGroupMemoryUsed(); - } - - /** - * Add a pinned batch - * @param batch Pinned batch to add - */ - public void addPinned(ManagedBatch batch) { - synchronized (this) { - addPinnedInternal(pinned, batch); - } - Map<TupleSourceID, Map<Integer, ManagedBatch>> theadPinned = PINNED_BY_THREAD.get(); - if (theadPinned == null) { - theadPinned = new HashMap<TupleSourceID, Map<Integer, ManagedBatch>>(); - PINNED_BY_THREAD.set(theadPinned); - } - addPinnedInternal(theadPinned, batch); - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE )) { - batch.captureCallStack(); - } - } - - private void addPinnedInternal(Map<TupleSourceID, Map<Integer, ManagedBatch>> pinnedMap, ManagedBatch batch) { - TupleSourceID tsID = batch.getTupleSourceID(); - Map<Integer, ManagedBatch> tsPinned = pinnedMap.get(tsID); - if(tsPinned == null) { - tsPinned = new HashMap<Integer, ManagedBatch>(); - pinnedMap.put(tsID, tsPinned); - } - - // Add batch, indexed by beginRow - tsPinned.put(new Integer(batch.getBeginRow()), batch); - } - - /** - * Remove a pinned batch, if not found do nothing and return null - * @param tsID Tuple source id - * @param beginRow Beginning row - * @return Removed batch or null if not found - */ - public ManagedBatch removePinned(TupleSourceID tsID, int beginRow) { - ManagedBatch result = null; - synchronized (this) { - result = removePinnedInternal(pinned, tsID, beginRow); - } - if (result != null) { - Map<TupleSourceID, Map<Integer, ManagedBatch>> theadPinned = PINNED_BY_THREAD.get(); - if (theadPinned != null) { - removePinnedInternal(theadPinned, tsID, beginRow); - } - } - return result; - } - - private ManagedBatch removePinnedInternal(Map<TupleSourceID, Map<Integer, ManagedBatch>> pinnedMap, TupleSourceID tsID, - int beginRow) { - Map<Integer, ManagedBatch> tsPinned = pinnedMap.get(tsID); - if(tsPinned != null) { - ManagedBatch mbatch = tsPinned.remove(new Integer(beginRow)); - - if(tsPinned.size() == 0) { - pinnedMap.remove(tsID); - } - - return mbatch; - } - return null; - } - - /** - * Add an unpinned batch - * @param batch Unpinned batch to add - */ - public void addUnpinned(ManagedBatch batch) { - unpinned.add(batch); - } - - /** - * Remove an unpinned batch - * @param batch Batch to remove - */ - public void removeUnpinned(ManagedBatch batch) { - unpinned.remove(batch); - } - - /** - * Get an iterator on all unpinned batches, typically for clean up - * purposes. This iterator is "safe" in that it is based on a copy - * of the real list and will not be invalidated by changes to the original - * list. However, this means that it also may contain batches that - * are no longer in the unpinned list, so the user of this iterator - * should check that each batch is still unpinned. - * @return Safe (but possibly out of date) iterator on unpinned batches - */ - public Iterator<ManagedBatch> getAllUnpinned() { - synchronized (unpinned) { - List<ManagedBatch> copy = new ArrayList<ManagedBatch>(unpinned); - return copy.iterator(); - } - } - - public synchronized Map<TupleSourceID, Map<Integer, ManagedBatch>> getAllPinned() { - return new HashMap<TupleSourceID, Map<Integer, ManagedBatch>>(pinned); - } - - public Map<TupleSourceID, Map<Integer, ManagedBatch>> getPinnedByCurrentThread() { - return PINNED_BY_THREAD.get(); - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -22,6 +22,7 @@ package com.metamatrix.common.buffer.impl; +import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -38,8 +39,8 @@ private String groupName; /** The bytes of memory used by this tuple group*/ - private long memoryUsed; - private Set<TupleSourceID> tupleSourceIDs = new HashSet<TupleSourceID>(); + private volatile long memoryUsed; + private Set<TupleSourceID> tupleSourceIDs = Collections.synchronizedSet(new HashSet<TupleSourceID>()); TupleGroupInfo(String groupName) { this.groupName = groupName; @@ -54,17 +55,14 @@ } long getGroupMemoryUsed() { - // no locking required. See MemoryState.NOTE1 return memoryUsed; } long reserveMemory(long bytes) { - // no locking required. See MemoryState.NOTE1 return memoryUsed += bytes; } long releaseMemory(long bytes) { - // no locking required. See MemoryState.NOTE1 return memoryUsed -= bytes; } } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -22,30 +22,55 @@ package com.metamatrix.common.buffer.impl; -import java.util.*; +import java.lang.ref.PhantomReference; +import java.lang.ref.ReferenceQueue; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; import com.metamatrix.common.types.DataTypeManager; +import com.metamatrix.common.types.Streamable; /** * Describe a TupleSource and all important information about it. */ public class TupleSourceInfo { + private static final AtomicLong LOB_ID = new AtomicLong(); + private static final ReferenceQueue<Streamable<?>> LOB_QUEUE = new ReferenceQueue<Streamable<?>>(); + + private static class LobReference extends PhantomReference<Streamable<?>> { + + String persistentStreamId; + + public LobReference(Streamable<?> lob) { + super(lob, LOB_QUEUE); + this.persistentStreamId = lob.getPersistenceStreamId(); + } + } + private TupleSourceType type; // Type of TupleSource, as defined in BufferManager constants private TupleSourceID tsID; private List schema; private String[] types; private int rowCount; - private TupleSourceStatus status; + private TupleSourceStatus status = TupleSourceStatus.ACTIVE; private TupleGroupInfo groupInfo; private boolean removed = false; private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>(); - + private Map<String, Streamable<?>> lobReferences; //references to contained lobs private boolean lobs; + @SuppressWarnings("unused") + private LobReference containingLobReference; //reference to containing lob + /** * Construct a TupleSourceInfo given information about it. * @param tsID Identifier @@ -58,16 +83,51 @@ this.schema = schema; this.types = types; this.groupInfo = groupInfo; - this.status = TupleSourceStatus.ACTIVE; - this.rowCount = 0; this.type = type; this.lobs = checkForLobs(); } + public void setContainingLobReference(Streamable<?> s) { + this.containingLobReference = new LobReference(s); + } + + public void addLobReference(Streamable<Object> lob) { + String id = lob.getReferenceStreamId(); + if (id == null) { + id = String.valueOf(LOB_ID.getAndIncrement()); + lob.setReferenceStreamId(id); + } + if (this.lobReferences == null) { + this.lobReferences = Collections.synchronizedMap(new HashMap<String, Streamable<?>>()); + } + this.lobReferences.put(id, lob); + } + + public static String getStaleLobTupleSource() { + LobReference ref = (LobReference)LOB_QUEUE.poll(); + if (ref == null) { + return null; + } + return ref.persistentStreamId; + } + + public Streamable<?> getLobReference(String id) { + if (this.lobReferences == null) { + return null; + } + return this.lobReferences.get(id); + } + public void addBatch(ManagedBatch batch) { batches.put(batch.getBeginRow(), batch); } + /** + * Returns the batch containing the begin row or null + * if it doesn't exist + * @param beginRow + * @return + */ public ManagedBatch getBatch(int beginRow) { Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(beginRow); if (entry != null && entry.getValue().getEndRow() >= beginRow) { @@ -184,19 +244,16 @@ return "TupleSourceInfo[" + this.tsID + "]"; //$NON-NLS-1$ //$NON-NLS-2$ } - private boolean checkForLobs() { - boolean lob = false; - if (types != null) { - for (int i = 0; i < types.length; i++) { - lob |= DataTypeManager.isLOB(types[i]); + if (types == null) { + // assume the worst + return true; + } + for (int i = 0; i < types.length; i++) { + if (DataTypeManager.isLOB(types[i]) || types[i] == DataTypeManager.DefaultDataTypes.OBJECT) { + return true; } } - else { - // if incase the user did not specify the types, then make - // them walk the batch; pay the penalty of performence - return true; - } - return lob; + return false; } } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -41,7 +41,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManagerPropertyNames; -import com.metamatrix.common.buffer.LobTupleBatch; import com.metamatrix.common.buffer.StorageManager; import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleSourceID; @@ -172,16 +171,6 @@ public void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types) throws MetaMatrixComponentException { - /* Right now we do not support the saving of the lobs to the disk. - * by throwing an exception the memory is never released for lobs, which is same - * as keeping them in a map. This is not going to be memory hog because, the actual - * lob (clob or blob) are backed by connector, xml is backed by already persisted - * tuple source. Here we are only saving the referenes to the actual objects. - */ - if (batch instanceof LobTupleBatch) { - throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStorageManager.can_not_save_lobs")); //$NON-NLS-1$ - } - // Defect 13342 - addBatch method now creates spill files if the total bytes exceeds the max file size limit TupleSourceInfo tsInfo = getTupleSourceInfo(sourceID, true); synchronized (tsInfo) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -181,7 +181,10 @@ } throw new MetaMatrixComponentException(e); } finally { - bufferMgr.releasePinnedBatches(); + //iff this is the root command context release any pinned (Unclosed tuplesources) + if (this.context.getParent() == null) { + bufferMgr.releasePinnedBatches(); + } } if(done || requestClosed) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/XMLSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/XMLSource.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/XMLSource.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -24,7 +24,6 @@ import java.io.StringReader; import java.util.List; -import java.util.Properties; import javax.xml.transform.Source; import javax.xml.transform.stream.StreamSource; @@ -32,10 +31,7 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.types.InvalidReferenceException; import com.metamatrix.common.types.XMLType; -import com.metamatrix.query.processor.xml.XMLUtil; /** @@ -59,12 +55,7 @@ // as processing excceptions. if (value instanceof XMLType) { XMLType xml = (XMLType)value; - try { - return xml.getSource(null); - } catch (InvalidReferenceException e) { - xml = XMLUtil.getFromBufferManager(bufferMgr, new TupleSourceID(xml.getPersistenceStreamId()), new Properties()); - return xml.getSource(null); - } + return xml.getSource(null); } return new StreamSource(new StringReader((String)value)); } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -561,7 +561,6 @@ String rsKey = rsName.toUpperCase(); CursorState state = this.cursorStates.remove(rsKey); if(state != null) { - state.ts.closeSource(); try { this.bufferMgr.removeTupleSource(state.tsID); } catch (TupleSourceNotFoundException e) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -36,7 +36,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.query.ExpressionEvaluationException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.rewriter.QueryRewriter; import com.metamatrix.query.sql.lang.AbstractSetCriteria; @@ -88,12 +87,6 @@ sortSymbols.add(dependentSetStates.get(i).valueExpression); } DependentValueSource originalVs = (DependentValueSource)dependentNode.getContext().getVariableContext().getGlobalValue(valueSource); - TupleSource ts; - try { - ts = dependentNode.getBufferManager().getTupleSource(originalVs.getTupleSourceID()); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); - } this.sortUtility = new SortUtility(originalVs.getTupleSourceID(), sortSymbols, sortDirection, true, dependentNode.getBufferManager(), dependentNode.getConnectionID()); } dvs = new DependentValueSource(sortUtility.sort(), dependentNode.getBufferManager()); Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -103,6 +103,7 @@ result.add(value); } } + its.closeSource(); if (cachedSets == null) { cachedSets = new HashMap<Expression, HashSet<Object>>(); } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -368,7 +368,7 @@ currentGroupTuple = null; groupBegin++; } - + this.groupTupleSource.closeSource(); if(rowCount != 0 || sortElements == null) { // Close last group List row = new ArrayList(functions.length); Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -299,6 +299,7 @@ matchEnd = -1; partitionedTuple = null; } + currentSource.closeSource(); currentSource = null; currentPartition++; } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -363,6 +363,7 @@ } finally { tc.saveBatch(); } + outTs.closeSource(); outTs = null; } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -71,7 +71,6 @@ import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.log.LogManager; import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.common.types.InvalidReferenceException; import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.Streamable; import com.metamatrix.common.types.XMLType; @@ -274,7 +273,7 @@ // if this is the first chunk, then create a tuple source id for this sequence of chunks if (!this.docInProgress) { this.docInProgress = true; - this.docInProgressTupleSourceId = XMLUtil.createXMLTupleSource(this.bufferMgr, this.resultsTupleSourceId.getStringID()); + this.docInProgressTupleSourceId = XMLUtil.createXMLTupleSource(this.bufferMgr, this.getContext().getConnectionID()); this.chunkPosition = 1; } @@ -287,8 +286,8 @@ // we want this to be naturally feed by chunks whether inside // or out side the processor - xml = new XMLType(); - xml.setPersistenceStreamId(this.docInProgressTupleSourceId.getStringID()); + xml = new XMLType(XMLUtil.getFromBufferManager(bufferMgr, this.docInProgressTupleSourceId, getProperties())); + this.bufferMgr.setPersistentTupleSource(this.docInProgressTupleSourceId, xml); //reset current document state. this.docInProgress = false; @@ -356,12 +355,7 @@ Reader source = null; try { - try { - source = xmlDoc.getCharacterStream(); - } catch (InvalidReferenceException e) { - xmlDoc = XMLUtil.getFromBufferManager(this.bufferMgr, new TupleSourceID(xmlDoc.getPersistenceStreamId()), props); - source = xmlDoc.getCharacterStream(); - } + source = xmlDoc.getCharacterStream(); // Validate against schema if(this.shouldValidate) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -32,18 +32,17 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManager; +import com.metamatrix.common.buffer.BufferManagerLobChunkStream; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; -import com.metamatrix.common.lob.BufferManagerLobChunkStream; import com.metamatrix.common.lob.ByteLobChunkStream; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.lob.LobChunkInputStream; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.XMLReaderFactory; -import com.metamatrix.common.types.XMLType; import com.metamatrix.query.sql.symbol.ElementSymbol; @@ -95,15 +94,8 @@ * This will reconstruct the XML object from the buffer manager from given * buffer manager id. */ - public static XMLType getFromBufferManager(final BufferManager bufferMgr, final TupleSourceID sourceId, Properties props) { - SQLXML sqlXML = new SQLXMLImpl(new BufferMangerXMLReaderFactory(bufferMgr, sourceId), props); - - // this is object to be sent to the client. The reference - // id will be set by the buffer manager. - XMLType xml = new XMLType(sqlXML); - xml.setPersistenceStreamId(sourceId.getStringID()); - - return xml; + public static SQLXML getFromBufferManager(final BufferManager bufferMgr, final TupleSourceID sourceId, Properties props) { + return new SQLXMLImpl(new BufferMangerXMLReaderFactory(bufferMgr, sourceId), props); } /** Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -160,21 +160,15 @@ List rows = new ArrayList(1); List row = new ArrayList(1); - // this may be little confusing, but the top layer is not immediately going - // to disk for saving; when it does it only saves the streaming id, not the - // contents. The below one saves immediately to disk, and when client refers to top - // id, processor know about the *saved* and gets the contents from it. + TupleSourceID savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.getContext().getConnectionID(), srcXML, this.chunkSize); + + //for large documents use the buffermanager version instead + if (this.bufferMgr.getFinalRowCount(savedId) > 1) { + srcXML = XMLUtil.getFromBufferManager(this.bufferMgr, savedId, getFormatProperties()); + } - // the one which saves to disk - TupleSourceID savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.resultsTupleSourceId.getStringID(), srcXML, this.chunkSize); - - // here we have 2 options; create xml from original source or from buffer - // manager. since buffer manager is slow we will choose the first option. - // incase this xml used in processor it will be faster; if it used in the - // client using steaming will be slow. XMLType xml = new XMLType(srcXML); - xml.setPersistenceStreamId(savedId.getStringID()); - //XMLValue xml = XMLUtil.getFromBufferManager(this.bufferMgr, savedId, this.chunkSize, getFormatProperties()); + this.bufferMgr.setPersistentTupleSource(savedId, xml); // now build the top batch with information from the saved one. row.add(xml); Modified: trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -30,18 +30,14 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.query.QueryProcessingException; -import com.metamatrix.common.util.PropertiesUtils; import com.metamatrix.core.util.ArgCheck; import com.metamatrix.query.QueryPlugin; import com.metamatrix.query.eval.SecurityFunctionEvaluator; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.optimizer.relational.PlanToProcessConverter; import com.metamatrix.query.processor.QueryProcessor; -import com.metamatrix.query.sql.symbol.ContextReference; import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.sql.symbol.Expression; -import com.metamatrix.query.sql.util.ValueIterator; -import com.metamatrix.query.sql.util.ValueIteratorSource; import com.metamatrix.query.sql.util.VariableContext; /** @@ -159,6 +155,10 @@ public CommandContext() { } + public CommandContext getParent() { + return parent; + } + public boolean isSessionFunctionEvaluated() { if (parent != null) { return parent.isSessionFunctionEvaluated(); Modified: trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/com/metamatrix/query/xquery/saxon/SaxonXQueryExpression.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -44,10 +44,7 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.types.SQLXMLImpl; -import com.metamatrix.core.util.StringUtil; import com.metamatrix.query.QueryPlugin; -import com.metamatrix.query.eval.Evaluator; -import com.metamatrix.query.sql.symbol.Expression; import com.metamatrix.query.util.XMLFormatConstants; import com.metamatrix.query.xquery.XQueryExpression; import com.metamatrix.query.xquery.XQuerySQLEvaluator; Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -27,17 +27,14 @@ import java.sql.SQLException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.lob.BufferManagerLobChunkStream; import com.metamatrix.common.lob.ByteLobChunkStream; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.lob.LobChunkProducer; import com.metamatrix.common.lob.ReaderInputStream; import com.metamatrix.common.types.BlobType; import com.metamatrix.common.types.ClobType; -import com.metamatrix.common.types.InvalidReferenceException; import com.metamatrix.common.types.Streamable; import com.metamatrix.common.types.XMLType; -import com.metamatrix.dqp.DQPPlugin; /** * A Lob Stream builder class. Given the Lob object this object can build @@ -48,7 +45,7 @@ LobChunkProducer internalStream = null; - public LobChunkStream(Streamable streamable, int chunkSize, BufferManager bufferMgr) + public LobChunkStream(Streamable<?> streamable, int chunkSize, BufferManager bufferMgr) throws IOException { try { @@ -64,14 +61,6 @@ BlobType blob = (BlobType)streamable; this.internalStream = new ByteLobChunkStream(blob.getBinaryStream(), chunkSize); } - } catch (InvalidReferenceException e) { - // if the lob did not have a persistent id, there is no way for us to re-create the - // object. so throw an error. - if (streamable.getPersistenceStreamId() == null) { - throw new IOException(DQPPlugin.Util.getString("LobStream.noreference")); //$NON-NLS-1$ - } - // otherwise read directly from the buffer manager. - this.internalStream = new BufferManagerLobChunkStream(streamable.getPersistenceStreamId(), bufferMgr); } catch(SQLException e) { IOException ex = new IOException(); ex.initCause(e); Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -23,14 +23,9 @@ package org.teiid.dqp.internal.process; import java.io.IOException; -import java.util.List; import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.CommonPlugin; import com.metamatrix.common.buffer.BlockedOnMemoryException; -import com.metamatrix.common.buffer.MemoryNotAvailableException; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.comm.api.ResultsReceiver; import com.metamatrix.common.lob.LobChunk; @@ -73,7 +68,7 @@ // If no previous stream is not found for this request create one and // save for future if (stream == null) { - stream = createLobStream(new TupleSourceID(streamId)); + stream = createLobStream(streamId); } // now get the chunk from stream @@ -119,35 +114,13 @@ * Create a object which can create a sequence of LobChunk objects on a given * LOB object */ - private LobChunkStream createLobStream(TupleSourceID referenceStreamId) + private LobChunkStream createLobStream(String referenceStreamId) throws BlockedOnMemoryException, MetaMatrixComponentException, IOException, TupleSourceNotFoundException { // get the reference object in the buffer manager, and try to stream off // the original sources. - TupleBatch batch = null; - try { - batch = dqpCore.getBufferManager().pinTupleBatch(referenceStreamId, 1, 1); - List[] tuples = batch.getAllTuples(); - - if (tuples != null && tuples.length > 0) { - Object anObj = tuples[0].get(0); - if (anObj instanceof Streamable) { - Streamable streamable = (Streamable)anObj; - return new LobChunkStream(streamable, chunkSize, dqpCore.getBufferManager()); - } - } - throw new MetaMatrixComponentException(DQPPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ - } catch (MemoryNotAvailableException e) { - throw BlockedOnMemoryException.INSTANCE; - } finally { - try { - if (batch != null) { - dqpCore.getBufferManager().unpinTupleBatch(referenceStreamId, batch.getBeginRow(), batch.getEndRow()); - } - } catch (MetaMatrixComponentException e) { - LogManager.logDetail(LogConstants.CTX_DQP, e, "Call to unpin failed during lob stream creation"); //$NON-NLS-1$ - } - } + Streamable<?> streamable = dqpCore.getBufferManager().getStreamable(parent.resultsID, referenceStreamId); + return new LobChunkStream(streamable, chunkSize, dqpCore.getBufferManager()); } synchronized void setResultsReceiver(ResultsReceiver<LobChunk> resultsReceiver) { Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -141,7 +141,7 @@ protected Command originalCommand; private AnalysisRecord analysisRecord; private TransactionContext transactionContext; - private TupleSourceID resultsID; + protected TupleSourceID resultsID; private Collection schemas; // These are schemas associated with XML results private boolean returnsUpdateCount; Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -236,19 +236,7 @@ assertTrue(xml1.getPersistenceStreamId() == null); assertTrue(xml2.getPersistenceStreamId() == null); - TupleSourceInfo info = mgr.getTupleSourceInfo(new TupleSourceID(xml1.getReferenceStreamId()), true); - // make sure the group name of the reference lob, is same as part batch id - assertEquals(id.getStringID(), info.getGroupInfo().getGroupName()); - - // now delete the parent tuple source, this should delete the - // all the kids with same name - mgr.removeTupleSource(id); - - try { - mgr.getTupleSource(new TupleSourceID(xml2.getReferenceStreamId())); - fail("this is already should have been cleaned up by above one"); //$NON-NLS-1$ - } catch (TupleSourceNotFoundException e) { - } + assertNotNull(mgr.getStreamable(id, xml1.getReferenceStreamId())); } public void testAddStreamablePart() throws Exception { Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -50,6 +50,7 @@ import com.metamatrix.common.buffer.BufferManagerFactory; import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.buffer.TupleSourceID; +import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.buffer.impl.BufferManagerImpl; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.XMLType; @@ -215,16 +216,18 @@ } } - private void helpProcessException(ProcessorPlan plan, ProcessorDataManager dataManager) { + private void helpProcessException(ProcessorPlan plan, ProcessorDataManager dataManager) throws TupleSourceNotFoundException, MetaMatrixComponentException { helpProcessException(plan, dataManager, null); } - private void helpProcessException(ProcessorPlan plan, ProcessorDataManager dataManager, String expectedErrorMessage) { - + private void helpProcessException(ProcessorPlan plan, ProcessorDataManager dataManager, String expectedErrorMessage) throws TupleSourceNotFoundException, MetaMatrixComponentException { + TupleSourceID tsId = null; + BufferManager bufferMgr = null; try { - BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); + bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); CommandContext context = new CommandContext("0", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager); + tsId = processor.getResultsID(); processor.process(); fail("Expected error during processing, but got none."); //$NON-NLS-1$ } catch(MetaMatrixCoreException e) { @@ -232,6 +235,8 @@ if(expectedErrorMessage != null) { assertEquals(expectedErrorMessage, e.getMessage()); } + } finally { + bufferMgr.removeTupleSource(tsId); } } @@ -2296,7 +2301,7 @@ * Tests a scalar subquery which returns more than one rows * causes the expected Exception */ - @Test public void testSubqueryScalarException() { + @Test public void testSubqueryScalarException() throws Exception { String sql = "SELECT e1, (SELECT e2 FROM pm2.g1) FROM pm1.g1"; //$NON-NLS-1$ // Construct data manager with data Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -107,13 +107,12 @@ } public static int getIntBatchSize() { - List[] expected = new List[] { - Arrays.asList(new Object[] { new Integer(0) }), - }; + List[] expected = new List[BATCH_SIZE]; + Arrays.fill(expected, Arrays.asList(1)); String[] types = { "integer" }; //$NON-NLS-1$ - int size = (int)SizeUtility.getBatchSize( types, expected ) * BATCH_SIZE; + int size = (int)SizeUtility.getBatchSize( types, expected ); return size; } Modified: trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java =================================================================== --- trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-08-12 16:36:33 UTC (rev 1234) +++ trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-08-12 17:39:14 UTC (rev 1235) @@ -54,7 +54,6 @@ private static final String DEFAULT_MANAGEMENT_INTERVAL = "0"; //$NON-NLS-1$ private static final String DEFAULT_LOG_STATS_INTERVAL = DEFAULT_MANAGEMENT_INTERVAL; private static final String DEFAULT_SESSION_USE_PERCENTAGE = "100"; //$NON-NLS-1$ - private static final String DEFAULT_ID_CREATOR = "com.metamatrix.common.buffer.impl.LongIDCreator"; //$NON-NLS-1$ private static final String DEFAULT_MAX_OPEN_FILES = "10"; //$NON-NLS-1$ // Instance

16 years, 8 months

1
0
0 / 0

teiid SVN: r1234 - in trunk/common-internal/src: test/java/com/metamatrix/common/log and 1 other directory.

by teiid-commits＠lists.jboss.org

Author: rareddy Date: 2009-08-12 12:36:33 -0400 (Wed, 12 Aug 2009) New Revision: 1234 Modified: trunk/common-internal/src/main/java/com/metamatrix/common/log/LogManager.java trunk/common-internal/src/test/java/com/metamatrix/common/log/TestLogManager.java Log: TEIID-770 Modified: trunk/common-internal/src/main/java/com/metamatrix/common/log/LogManager.java =================================================================== --- trunk/common-internal/src/main/java/com/metamatrix/common/log/LogManager.java 2009-08-11 03:25:47 UTC (rev 1233) +++ trunk/common-internal/src/main/java/com/metamatrix/common/log/LogManager.java 2009-08-12 16:36:33 UTC (rev 1234) @@ -109,7 +109,7 @@ * not logged if this parameter is null */ public static void logCritical(String context, String message) { - logListener.log(MessageLevel.CRITICAL, context, message); + logMessage(MessageLevel.CRITICAL, context, message); } /** @@ -128,7 +128,7 @@ * @param message the log message (may be null) */ public static void logCritical(String context, Throwable e, String message) { - logListener.log(MessageLevel.CRITICAL,context,e,message); + log(MessageLevel.CRITICAL,context,e,message); } /** @@ -144,7 +144,7 @@ * not logged if this parameter is null */ public static void logError(String context, String message) { - logListener.log(MessageLevel.ERROR, context,message); + logMessage(MessageLevel.ERROR, context,message); } /** @@ -161,7 +161,7 @@ * @param message the log message (may be null) */ public static void logError(String context, Throwable e, String message) { - logListener.log(MessageLevel.ERROR,context,e,message); + log(MessageLevel.ERROR,context,e,message); } /** @@ -177,7 +177,7 @@ * not logged if this parameter is null */ public static void logWarning(String context, String message) { - logListener.log(MessageLevel.WARNING, context,message); + logMessage(MessageLevel.WARNING, context,message); } /** @@ -194,7 +194,7 @@ * @param message the log message (may be null) */ public static void logWarning(String context, Throwable e, String message) { - logListener.log(MessageLevel.WARNING,context,e,message); + log(MessageLevel.WARNING,context,e,message); } /** @@ -211,7 +211,7 @@ * not logged if this parameter is null */ public static void logInfo(String context, String message) { - logListener.log(MessageLevel.INFO, context,message); + logMessage(MessageLevel.INFO, context,message); } /** @@ -244,7 +244,7 @@ * @param message the log message (may be null) */ public static void logDetail(String context, Throwable e, String message) { - logListener.log(MessageLevel.DETAIL,context,e,message); + log(MessageLevel.DETAIL,context,e,message); } /** @@ -294,7 +294,7 @@ * not logged if this parameter is null */ public static void log(int msgLevel, String context, Object message) { - logListener.log(msgLevel, context, message); + logMessage(msgLevel, context, message); } /** @@ -310,6 +310,9 @@ * not logged if this parameter is null */ public static void log(int msgLevel, String context, Throwable e, Object message) { + if (!isMessageToBeRecorded(context, msgLevel)) { + return; + } logListener.log(msgLevel, context, e, message); } Modified: trunk/common-internal/src/test/java/com/metamatrix/common/log/TestLogManager.java =================================================================== --- trunk/common-internal/src/test/java/com/metamatrix/common/log/TestLogManager.java 2009-08-11 03:25:47 UTC (rev 1233) +++ trunk/common-internal/src/test/java/com/metamatrix/common/log/TestLogManager.java 2009-08-12 16:36:33 UTC (rev 1234) @@ -98,16 +98,57 @@ } List recevedMsgList = listener.getLoggedMessages(); + assertEquals(sentMsgList.size(), recevedMsgList.size()); assertEquals(sentMsgList, recevedMsgList); } + + public void testFilterMessage() { + final LogConfiguration cfg = LogManager.getLogConfigurationCopy(); + cfg.setLogLevel(context, MessageLevel.INFO ); + LogManager.setLogConfiguration(cfg); + + LogListener listener = new LogListener() { + @Override + public void log(int level, String context, Object msg) { + int setLevel = cfg.getLogLevel(context); + assertTrue(level <= setLevel); + } + @Override + public void log(int level, String context, Throwable t, Object msg) { + int setLevel = cfg.getLogLevel(context); + assertTrue(level <= setLevel); + } + + @Override + public void shutdown() { + } + }; + LogManager.logListener = listener; + + LogManager.logCritical(context, "message"); //$NON-NLS-1$ + LogManager.logDetail(context, "msgParts");//$NON-NLS-1$ + LogManager.logError(context, "message");//$NON-NLS-1$ + LogManager.logInfo(context, "message");//$NON-NLS-1$ + LogManager.logTrace(context, "msgParts");//$NON-NLS-1$ + LogManager.logWarning(context, "message");//$NON-NLS-1$ + + Exception e = new Exception("test"); //$NON-NLS-1$ + LogManager.logCritical(context, e, "message"); //$NON-NLS-1$ + LogManager.logDetail(context, e, "msgParts");//$NON-NLS-1$ + LogManager.logError(context, e, "message");//$NON-NLS-1$ + LogManager.log(MessageLevel.INFO, context, e, "message");//$NON-NLS-1$ + LogManager.logTrace(context,e, "msgParts");//$NON-NLS-1$ + LogManager.logWarning(context, e, "message");//$NON-NLS-1$ + } + /** * * A log listener that saves messages (IStatus)s in a * List for later comparison. */ class ListLogger implements LogListener { - private List messages = new ArrayList(); + private List<String> messages = new ArrayList<String>(); private int expectedMessages; public ListLogger(int expectedMessages) { @@ -118,14 +159,14 @@ * @see com.metamatrix.core.log.LogListener#logMessage(org.eclipse.core.runtime.IStatus, long, java.lang.String, java.lang.String) */ public synchronized void log(int level, String context, Object msg){ - this.messages.add(msg); + this.messages.add(msg.toString()); if (this.messages.size() == expectedMessages) { this.notifyAll(); } } public void log(int level, String context, Throwable t, Object msg) { - this.messages.add(msg); + this.messages.add(msg.toString()); if (this.messages.size() == expectedMessages) { this.notifyAll(); } @@ -153,6 +194,4 @@ } - - }

16 years, 8 months

1
0
0 / 0

Teiid-NightlyBuild - Build # 541 - Fixed!

by jboss-qa-internal＠redhat.com

Teiid-NightlyBuild - Build # 541 - Fixed: Check console output at http://hudson.qa.jboss.com/hudson/job/Teiid-NightlyBuild/541/ to view the results.

16 years, 8 months

1
0
0 / 0

teiid SVN: r1233 - in trunk: engine/src/main/java/com/metamatrix/common/buffer/impl and 10 other directories.

by teiid-commits＠lists.jboss.org

Author: shawkins Date: 2009-08-10 23:25:47 -0400 (Mon, 10 Aug 2009) New Revision: 1233 Added: trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java Removed: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerFactory.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMap.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMapValueTranslator.java trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/memory/ trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/memory/ Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerPropertyNames.java trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBatchMap.java trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java trunk/server/src/main/java/com/metamatrix/server/dqp/service/PlatformBufferService.java Log: TEIID-767 merging memorystoremanager directly into the buffermanager. Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -90,12 +90,10 @@ int getConnectorBatchSize(); /** - * Adds a {@link StorageManager} to this BufferManager instance. This - * method may be called multiple times; it will be first called after the - * call to {@link #initialize}. + * Adds a {@link StorageManager} to this BufferManager instance. * @param storageManager Storage manager to add */ - void addStorageManager(StorageManager storageManager); + void setStorageManager(StorageManager storageManager); /** * Creates a tuple source based on a schema and properties describing Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerFactory.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -1,103 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -import java.util.Properties; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.common.buffer.storage.file.FileStorageManager; -import com.metamatrix.common.buffer.storage.memory.MemoryStorageManager; -import com.metamatrix.common.util.PropertiesUtils; - -/** - * Factory for BufferManager instances. One method will get - * a server buffer manager, as it should be instantiated in a running - * MetaMatrix server. That BufferManager is configured mostly by the - * passed in properties. - * - * The other method returns a stand-alone, in-memory buffer manager. This - * is typically used for either in-memory testing or any time the - * query processor component is not expected to run out of memory, such as - * within the modeler. - */ -public class BufferManagerFactory { - - private static BufferManager INSTANCE; - - /** - * Helper to get a buffer manager all set up for unmanaged standalone use. This is - * typically used for testing or when memory is not an issue. - * @return BufferManager ready for use - */ - public static BufferManager getStandaloneBufferManager() throws MetaMatrixComponentException { - if (INSTANCE == null) { - BufferManager bufferMgr = new BufferManagerImpl(); - Properties props = new Properties(); - props.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, String.valueOf(Long.MAX_VALUE)); - props.setProperty(BufferManagerPropertyNames.SESSION_USE_PERCENTAGE, "100"); //$NON-NLS-1$ - props.setProperty(BufferManagerPropertyNames.LOG_STATS_INTERVAL, "0"); //$NON-NLS-1$ - props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$ - bufferMgr.initialize("local", props); //$NON-NLS-1$ - - // Add unmanaged memory storage manager - bufferMgr.addStorageManager(new MemoryStorageManager()); - bufferMgr.addStorageManager(new MemoryStorageManager() { - @Override - public int getStorageType() { - return StorageManager.TYPE_FILE; - } - }); - INSTANCE = bufferMgr; - } - - return INSTANCE; - } - - /** - * Helper to get a buffer manager all set up for unmanaged standalone use. This is - * typically used for testing or when memory is not an issue. - * @param lookup Lookup implementation to use - * @param props Configuration properties - * @return BufferManager ready for use - */ - public static BufferManager getServerBufferManager(String lookup, Properties props) throws MetaMatrixComponentException { - Properties bmProps = PropertiesUtils.clone(props, false); - // Construct buffer manager - BufferManager bufferManager = new BufferManagerImpl(); - bufferManager.initialize(lookup, bmProps); - - // Get the properties for FileStorageManager and create. - StorageManager fsm = new FileStorageManager(); - fsm.initialize(bmProps); - bufferManager.addStorageManager(fsm); - - // Create MemoryStorageManager - StorageManager msm = new MemoryStorageManager(); - msm.initialize(bmProps); - bufferManager.addStorageManager(msm); - - return bufferManager; - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerPropertyNames.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerPropertyNames.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerPropertyNames.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -28,12 +28,6 @@ public final class BufferManagerPropertyNames { /** - * Optional property - the class name of the memory manager to use (must be an implementation - * of {@link com.metamatrix.buffer.impl.BufferIDCreator}. - */ - public static final String ID_CREATOR = "metamatrix.buffer.idCreator"; //$NON-NLS-1$ - - /** * Optional property - the amount of memory (in megabytes) that buffer management should use. * This value should be a positive integer (less than the the max heap size) and defaults to * 128. Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -39,11 +39,6 @@ public interface StorageManager { /** - * Constant for a StorageManager for in-memory storage - */ - public static final int TYPE_MEMORY = 0; - - /** * Constant for a StorageManager for database storage */ public static final int TYPE_DATABASE = 1; @@ -68,16 +63,6 @@ throws MetaMatrixComponentException; /** - * Get the type of storage as defined by constants. - * @return Storage type - * @see StorageManager#TYPE_MEMORY - * @see StorageManager#TYPE_DATABASE - * @see StorageManager#TYPE_FILE - * @see StorageManager#TYPE_REMOTE - */ - int getStorageType(); - - /** * Add a batch to the storage manager. * @param types a hint to the StorageManager about the types of data in the batch * @throws MetaMatrixComponentException indicating a non-business-related @@ -99,17 +84,6 @@ throws TupleSourceNotFoundException, MetaMatrixComponentException; /** - * Remove a batch from this storage as specified. If the tuple source - * is unknown or the batch is unknown, a TupleSourceNotFoundException is - * thrown. - * @throws TupleSourceNotFoundException indicating the sourceID is unknown - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void removeBatch(TupleSourceID sourceID, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** * Remove all batches for the specified tuple source. If no batches exist, * no exception is thrown. * @throws MetaMatrixComponentException indicating a non-business-related Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMap.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMap.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMap.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -1,107 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.TreeMap; - - -/** - * This class is used to save batches ordered by the first row. - * @since 4.3 - */ -public class BatchMap { - private TreeMap batches; - private BatchMapValueTranslator translator; - - public BatchMap(BatchMapValueTranslator translator) { - batches = new TreeMap(); - this.translator = translator; - } - - public void addBatch(int beginRow, Object batch) { - batches.put(new Integer(beginRow), batch); - } - - public Object getBatch(int beginRow) { - - //check special cases first - int batchSize = batches.size(); - if(batchSize == 0) { - return null; - } - Object batch; - if(batchSize == 1) { - batch = batches.values().iterator().next(); - if(beginRow >= translator.getBeginRow(batch) && beginRow <= translator.getEndRow(batch)) { - return batch; - } - return null; - } - - //Try to search by beginRow. This is fast. - if((batch = batches.get(new Integer(beginRow))) != null) { - return batch; - } - - List batchList = new ArrayList(batches.values()); - return doBinarySearchForBatch(batchList, beginRow); - } - - public void removeBatch(int beginRow) { - if(batches.isEmpty()) { - return; - } - batches.remove(new Integer(beginRow)); - } - - public Iterator getBatchIterator() { - return batches.values().iterator(); - } - - public boolean isEmpty() { - return batches.isEmpty(); - } - - private Object doBinarySearchForBatch(List batches, int beginRow) { - int batchSize = batches.size(); - int beginIndex = 0; - int midIndex; - int endIndex = batchSize; - Object batch; - while(beginIndex < endIndex) { - midIndex = (beginIndex + endIndex)/2; - batch = batches.get(midIndex); - if(beginRow < translator.getBeginRow(batch)){ - endIndex = midIndex; - }else if(beginRow > translator.getEndRow(batch)) { - beginIndex = midIndex + 1; - }else { - return batch; - } - } - return null; - } -} Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMapValueTranslator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMapValueTranslator.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BatchMapValueTranslator.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -1,45 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - - -/** - * @since 4.3 - */ -public interface BatchMapValueTranslator { - /** - * Return the beginning row number of the given batch. - * @param batchMapValue - * @return - * @since 4.3 - */ - int getBeginRow(Object batchMapValue); - - /** - * Return the ending row number of the given batch. - * @param batchMapValue - * @return - * @since 4.3 - */ - int getEndRow(Object batchMapValue); -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -77,7 +77,6 @@ private Map<String, TupleGroupInfo> groupInfos = new HashMap<String, TupleGroupInfo>(); // Storage managers - private StorageManager memoryMgr; private StorageManager diskMgr; // ID creator @@ -174,9 +173,9 @@ } synchronized(info) { - Iterator batchIter = info.getBatchIterator(); + Iterator<ManagedBatch> batchIter = info.getBatchIterator(); while(batchIter.hasNext()) { - ManagedBatch batch = (ManagedBatch) batchIter.next(); + ManagedBatch batch = batchIter.next(); switch(batch.getLocation()) { case ManagedBatch.PERSISTENT: stats.numPersistentBatches++; @@ -226,14 +225,10 @@ * Add a storage manager to this buffer manager, order is unimportant * @param storageManager Storage manager to add */ - public void addStorageManager(StorageManager storageManager) { + public void setStorageManager(StorageManager storageManager) { Assertion.isNotNull(storageManager); - if(storageManager.getStorageType() == StorageManager.TYPE_MEMORY) { - this.memoryMgr = storageManager; - } else { - Assertion.isNull(diskMgr); - this.diskMgr = storageManager; - } + Assertion.isNull(diskMgr); + this.diskMgr = storageManager; } /** @@ -289,9 +284,9 @@ if(! info.isRemoved()) { info.setRemoved(); - Iterator iter = info.getBatchIterator(); + Iterator<ManagedBatch> iter = info.getBatchIterator(); while(iter.hasNext()) { - ManagedBatch batch = (ManagedBatch) iter.next(); + ManagedBatch batch = iter.next(); switch(batch.getLocation()) { case ManagedBatch.UNPINNED: memoryState.removeUnpinned(batch); @@ -309,9 +304,7 @@ synchronized (tupleGroupInfo) { tupleGroupInfo.getTupleSourceIDs().remove(tupleSourceID); } - // Remove memory storage - this.memoryMgr.removeBatches(tupleSourceID); - + // Remove disk storage if (this.diskMgr != null){ this.diskMgr.removeBatches(tupleSourceID); @@ -491,12 +484,16 @@ throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$ } + // Update tuple source state + ManagedBatch managedBatch = new ManagedBatch(tupleSourceID, tupleBatch.getBeginRow(), tupleBatch.getEndRow(), bytes); + managedBatch.setLocation(location); + // Store into storage manager try { if(location == ManagedBatch.PERSISTENT) { this.diskMgr.addBatch(tupleSourceID, tupleBatch, info.getTypes()); } else { - this.memoryMgr.addBatch(tupleSourceID, tupleBatch, info.getTypes()); + managedBatch.setBatch(tupleBatch); } } catch(MetaMatrixComponentException e) { // If we were storing to memory, clean up memory we reserved @@ -505,11 +502,7 @@ } throw e; } - - // Update tuple source state - ManagedBatch managedBatch = new ManagedBatch(tupleSourceID, tupleBatch.getBeginRow(), tupleBatch.getEndRow(), bytes); - managedBatch.setLocation(location); - + // Add to memory state if in memory if(location == ManagedBatch.UNPINNED) { this.memoryState.addUnpinned(managedBatch); @@ -565,7 +558,7 @@ } else if(mbatch.getLocation() == ManagedBatch.PINNED) { // Load batch from memory - already pinned - memoryBatch = this.memoryMgr.getBatch(tupleSourceID, beginRow, info.getTypes()); + memoryBatch = mbatch.getBatch(); } else if(mbatch.getLocation() == ManagedBatch.UNPINNED) { // Already in memory - just move from unpinned to pinned @@ -574,7 +567,7 @@ this.memoryState.addPinned(mbatch); // Load batch from memory - memoryBatch = this.memoryMgr.getBatch(tupleSourceID, beginRow, info.getTypes()); + memoryBatch = mbatch.getBatch(); } else if(mbatch.getLocation() == ManagedBatch.PERSISTENT) { memoryRequiredByBatch = mbatch.getSize(); @@ -601,26 +594,7 @@ int internalBeginRow = mbatch.getBeginRow(); memoryBatch = diskMgr.getBatch(tupleSourceID, internalBeginRow, info.getTypes()); - try { - memoryMgr.addBatch(tupleSourceID, memoryBatch, info.getTypes()); - } catch(MetaMatrixComponentException e) { - memoryState.releaseMemory(mbatch.getSize(), info.getGroupInfo()); - throw e; - } - - try { - diskMgr.removeBatch(tupleSourceID, internalBeginRow); - } catch(TupleSourceNotFoundException e) { - } catch(MetaMatrixComponentException e) { - memoryState.releaseMemory(memoryRequiredByBatch, info.getGroupInfo()); - try { - memoryMgr.removeBatch(tupleSourceID, internalBeginRow); - } catch(Exception e2) { - // ignore - } - throw e; - } - + mbatch.setBatch(memoryBatch); mbatch.setLocation(ManagedBatch.PINNED); this.memoryState.addPinned(mbatch); } @@ -736,13 +710,13 @@ long totalMemory = config.getTotalAvailableMemory(); long released = 0; - Iterator unpinnedIter = this.memoryState.getAllUnpinned(); + Iterator<ManagedBatch> unpinnedIter = this.memoryState.getAllUnpinned(); while(unpinnedIter.hasNext() && // If there are unpinned batches in memory, AND // Defect 14573 - if we require more than what's available, then cleanup regardless of the threshold (memoryRequired > totalMemory - memoryState.getMemoryUsed() || // if the memory needed is more than what's available, or memoryState.getMemoryUsed() > targetLevel)){ // if we've crossed the active memory threshold, then cleanup - ManagedBatch batch = (ManagedBatch) unpinnedIter.next(); + ManagedBatch batch = unpinnedIter.next(); TupleSourceID tsID = batch.getTupleSourceID(); released += releaseMemory(batch, tsID); @@ -763,9 +737,9 @@ boolean cleanForSessionSucceeded = false; long released = 0; - Iterator unpinnedIter = this.memoryState.getAllUnpinned(); + Iterator<ManagedBatch> unpinnedIter = this.memoryState.getAllUnpinned(); while(unpinnedIter.hasNext()) { - ManagedBatch batch = (ManagedBatch) unpinnedIter.next(); + ManagedBatch batch = unpinnedIter.next(); TupleSourceID tsID = batch.getTupleSourceID(); TupleSourceInfo tsInfo = getTupleSourceInfo(tsID, false); if(tsInfo == null) { @@ -828,15 +802,7 @@ } // This batch is still unpinned - move to persistent storage - int beginRow = batch.getBeginRow(); - TupleBatch dataBatch = null; - try { - dataBatch = memoryMgr.getBatch(tsID, beginRow, info.getTypes()); - } catch(TupleSourceNotFoundException e) { - return 0; - } catch(MetaMatrixComponentException e) { - return 0; - } + TupleBatch dataBatch = batch.getBatch(); try { diskMgr.addBatch(tsID, dataBatch, info.getTypes()); @@ -845,13 +811,7 @@ return 0; } - try { - memoryMgr.removeBatch(tsID, beginRow); - } catch(TupleSourceNotFoundException e) { - // ignore - } catch(MetaMatrixComponentException e) { - // ignore - } + batch.setBatch(null); // Update memory batch.setLocation(ManagedBatch.PERSISTENT); @@ -1048,18 +1008,18 @@ * @see com.metamatrix.common.buffer.BufferManager#releasePinnedBatches() */ public void releasePinnedBatches() throws MetaMatrixComponentException { - Map threadPinned = memoryState.getPinnedByCurrentThread(); + Map<TupleSourceID, Map<Integer, ManagedBatch>> threadPinned = memoryState.getPinnedByCurrentThread(); if (threadPinned == null) { return; } - for (Iterator i = threadPinned.entrySet().iterator(); i.hasNext();) { - Map.Entry entry = (Map.Entry)i.next(); + for (Iterator<Map.Entry<TupleSourceID, Map<Integer, ManagedBatch>>> i = threadPinned.entrySet().iterator(); i.hasNext();) { + Map.Entry<TupleSourceID, Map<Integer, ManagedBatch>> entry = i.next(); i.remove(); - TupleSourceID tsid = (TupleSourceID)entry.getKey(); - Map pinnedBatches = (Map)entry.getValue(); + TupleSourceID tsid = entry.getKey(); + Map<Integer, ManagedBatch> pinnedBatches = entry.getValue(); try { - for (Iterator j = pinnedBatches.values().iterator(); j.hasNext();) { - ManagedBatch batch = (ManagedBatch)j.next(); + for (Iterator<ManagedBatch> j = pinnedBatches.values().iterator(); j.hasNext();) { + ManagedBatch batch = j.next(); //TODO: add trace logging about the batch that is being unpinned unpinTupleBatch(tsid, batch.getBeginRow(), batch.getEndRow()); @@ -1074,7 +1034,7 @@ * for testing purposes */ public int getPinnedCount() { - Map pinned = memoryState.getAllPinned(); + Map<TupleSourceID, Map<Integer, ManagedBatch>> pinned = memoryState.getAllPinned(); int count = 0; @@ -1082,8 +1042,8 @@ return count; } - for (Iterator i = pinned.values().iterator(); i.hasNext();) { - count += ((Map)i.next()).size(); + for (Iterator<Map<Integer, ManagedBatch>> i = pinned.values().iterator(); i.hasNext();) { + count += i.next().size(); } return count; Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -26,6 +26,7 @@ import java.util.LinkedList; import java.util.List; +import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.core.util.HashCodeUtil; @@ -56,12 +57,13 @@ private int location; private int pinnedCount; private int hashCode; + private TupleBatch batch; // logging private static int STACK_LEVELS_TO_OMIT = 2; private static int STACK_LEVELS_TO_CAPTURE = 5; - private List whoCalledUs; + private List<String> whoCalledUs; private String sCallStackTimeStamp; /** @@ -75,6 +77,14 @@ this.hashCode = HashCodeUtil.hashCode(tupleSourceID.hashCode(), beginRow); } + public TupleBatch getBatch() { + return batch; + } + + public void setBatch(TupleBatch batch) { + this.batch = batch; + } + /** * Get the tuple source ID * @return Tuple sourceID @@ -147,7 +157,7 @@ */ StackTraceElement[] elements = new Exception().getStackTrace(); - whoCalledUs = new LinkedList(); + whoCalledUs = new LinkedList<String>(); for ( int i = STACK_LEVELS_TO_OMIT; i < elements.length && i < STACK_LEVELS_TO_OMIT + STACK_LEVELS_TO_CAPTURE; i++ ) { whoCalledUs.add(elements[ i ].toString()); @@ -158,7 +168,7 @@ /** * Returns call stack */ - public List getCallStack() { + public List<String> getCallStack() { return whoCalledUs; } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -34,15 +34,6 @@ */ public class TupleSourceInfo { - private static final BatchMapValueTranslator TRANSLATOR = new BatchMapValueTranslator() { - public int getBeginRow(Object batchMapValue) { - return ((ManagedBatch)batchMapValue).getBeginRow(); - } - public int getEndRow(Object batchMapValue) { - return ((ManagedBatch)batchMapValue).getEndRow(); - } - }; - private TupleSourceType type; // Type of TupleSource, as defined in BufferManager constants private TupleSourceID tsID; private List schema; @@ -51,8 +42,8 @@ private TupleSourceStatus status; private TupleGroupInfo groupInfo; private boolean removed = false; + private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>(); - private BatchMap batches; private boolean lobs; /** @@ -63,7 +54,6 @@ * @param type Type of tuple source, as defined in BufferManager constants */ public TupleSourceInfo(TupleSourceID tsID, List schema, String[] types, TupleGroupInfo groupInfo, TupleSourceType type) { - this.tsID = tsID; this.schema = schema; this.types = types; @@ -71,10 +61,29 @@ this.status = TupleSourceStatus.ACTIVE; this.rowCount = 0; this.type = type; - this.batches = new BatchMap(TRANSLATOR); this.lobs = checkForLobs(); } + public void addBatch(ManagedBatch batch) { + batches.put(batch.getBeginRow(), batch); + } + + public ManagedBatch getBatch(int beginRow) { + Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(beginRow); + if (entry != null && entry.getValue().getEndRow() >= beginRow) { + return entry.getValue(); + } + return null; + } + + public void removeBatch(int beginRow) { + batches.remove(new Integer(beginRow)); + } + + public Iterator<ManagedBatch> getBatchIterator() { + return batches.values().iterator(); + } + /** * Get the tuple source identifier * @return Tuple source identifier @@ -159,47 +168,10 @@ this.removed = true; } - /** - * Add a managed batch to this tuple source - * @param batch Managed batch for this tuple source - */ - public void addBatch(ManagedBatch batch) { - batches.addBatch(batch.getBeginRow(), batch); - } - - /** - * Remove a managed batch for this tuple source, - * indexed by beginRow - * @param beginRow First row of batch to remove - */ - public void removeBatch(int beginRow) { - batches.removeBatch(beginRow); - } - - /** - * Get a batch, specified by beginRow - * @param beginRow First row of batch to retrieve - * @return Managed batch or null if not found - */ - public ManagedBatch getBatch(int beginRow) { - return (ManagedBatch)batches.getBatch(beginRow); - } - public boolean lobsInSource() { return this.lobs; } - /** - * Get iterator directly on the batches contained by this - * tuple source. To use this safely, the user should ensure - * through locking that no other thread are modifying this - * TupleSourceInfo while the iterator is being used. - * @return Unsafe iterator - */ - public Iterator getBatchIterator() { - return this.batches.getBatchIterator(); - } - public String[] getTypes() { return types; } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -33,8 +33,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -72,7 +72,7 @@ private File dirFile; // State - private Map tupleSourceMap = new HashMap(); // TupleSourceID -> TupleSourceInfo + private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new HashMap<TupleSourceID, TupleSourceInfo>(); private Map<File, RandomAccessFile> fileCache = Collections.synchronizedMap(new LinkedHashMap<File, RandomAccessFile>() { @Override protected boolean removeEldestEntry( @@ -119,14 +119,6 @@ } /** - * Return file type: {@link com.metamatrix.common.buffer.StorageManager.TYPE_FILE} - * @return File type constant - */ - public int getStorageType() { - return StorageManager.TYPE_FILE; - } - - /** * Look up tuple source info and possibly create. First the file map is used to find an * existing file info. If the info is found it is returned. If not, then * a TupleSourceInfo is created according to shouldCreate flag @@ -138,7 +130,7 @@ // Try to find in cache synchronized(tupleSourceMap) { - TupleSourceInfo info = (TupleSourceInfo) tupleSourceMap.get(sourceID); + TupleSourceInfo info = tupleSourceMap.get(sourceID); if(info == null && shouldCreate) { info = new TupleSourceInfo(); tupleSourceMap.put(sourceID, info); @@ -285,7 +277,7 @@ throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ } // Find pointer - PointerInfo pointerInfo = (PointerInfo) info.tupleBatchPointers.get(new Integer(beginRow)); + PointerInfo pointerInfo = info.tupleBatchPointers.get(new Integer(beginRow)); Assertion.isNotNull(pointerInfo); FileInfo fileInfo = pointerInfo.fileInfo; @@ -305,9 +297,9 @@ batch.readExternal(ois); return batch; } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ } catch (ClassNotFoundException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ } finally { fileInfo.close(); } @@ -315,19 +307,6 @@ } /** - * This method does nothing - rather than deleting batches from the middle of a RandomAccessFile, - * which would be very expensive, we just handle the possibility that a batch already exists - * in the addBatch method. - * @param sourceID Source identifier - * @param beginRow Beginning batch row to remove - */ - public void removeBatch(TupleSourceID sourceID, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - // nothing - don't remove batches as it is too expensive - } - - /** * Remove all batches for a sourceID. Before removal, the file is closed. * @param sourceID Tuple source ID */ @@ -335,7 +314,7 @@ TupleSourceInfo info = null; // Remove info from the file map synchronized(tupleSourceMap) { - info = (TupleSourceInfo)tupleSourceMap.remove(sourceID); + info = tupleSourceMap.remove(sourceID); } // Didn't find a file @@ -355,7 +334,7 @@ // If open, close the file and decrement the open file counter for (int i = 0; i < info.storageFiles.size(); i++) { - FileInfo fileInfo = (FileInfo)info.storageFiles.get(i); + FileInfo fileInfo = info.storageFiles.get(i); fileInfo.delete(); } // Delete the file and mark info as being removed @@ -371,10 +350,12 @@ LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing all storage area files "); //$NON-NLS-1$ - Iterator tsIter = tupleSourceMap.keySet().iterator(); - - while(tsIter.hasNext()) { - TupleSourceID key = (TupleSourceID)tsIter.next(); + List<TupleSourceID> ids = null; + synchronized (tupleSourceMap) { + ids = new LinkedList<TupleSourceID>(tupleSourceMap.keySet()); + } + + for (TupleSourceID key : ids) { try { removeBatches(key); } catch (MetaMatrixComponentException e) { @@ -398,10 +379,6 @@ this.file = file; } - public boolean isOpen() { - return fileData != null; - } - public void open() throws FileNotFoundException { if(this.fileData == null) { this.fileData = fileCache.remove(this.file); @@ -454,15 +431,15 @@ } private static class TupleSourceInfo { - Map tupleBatchPointers = new HashMap(); // beginRow -> PointerInfo - List storageFiles = new ArrayList(2); // Stores all the FileInfos for this tupleSource + Map<Integer, PointerInfo> tupleBatchPointers = new HashMap<Integer, PointerInfo>(); + List<FileInfo> storageFiles = new ArrayList<FileInfo>(2); // Stores all the FileInfos for this tupleSource private boolean isRemoved = false; FileInfo getMostRecentlyCreatedFile() { if (storageFiles.isEmpty()) { return null; } - return (FileInfo)storageFiles.get(storageFiles.size() - 1); + return storageFiles.get(storageFiles.size() - 1); } } Copied: trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java (from rev 1214, trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerFactory.java) =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java (rev 0) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -0,0 +1,141 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.common.buffer.impl.BufferManagerImpl; +import com.metamatrix.query.execution.QueryExecPlugin; + +/** + * Factory for BufferManager instances. One method will get + * a server buffer manager, as it should be instantiated in a running + * MetaMatrix server. That BufferManager is configured mostly by the + * passed in properties. + * + * The other method returns a stand-alone, in-memory buffer manager. This + * is typically used for either in-memory testing or any time the + * query processor component is not expected to run out of memory, such as + * within the modeler. + */ +public class BufferManagerFactory { + + public static class MemoryStorageManager implements StorageManager { + + // TupleSourceID -> List<TupleBatch> (ordered by startRow) + private Map<TupleSourceID, Map<Integer, TupleBatch>> storage = Collections.synchronizedMap(new HashMap<TupleSourceID, Map<Integer, TupleBatch>>()); + + /** + * @see StorageManager#initialize(Properties) + */ + public void initialize(Properties props) throws MetaMatrixComponentException { + } + + /** + * @see StorageManager#addBatch(TupleSourceID, TupleBatch) + */ + public void addBatch(TupleSourceID storageID, TupleBatch batch, String[] types) + throws MetaMatrixComponentException { + + Map<Integer, TupleBatch> batches = null; + synchronized(this.storage) { + batches = storage.get(storageID); + if(batches == null) { + batches = new HashMap<Integer, TupleBatch>(); + this.storage.put(storageID, batches); + } + } + + synchronized(batches) { + batches.put(batch.getBeginRow(), batch); + } + } + + /** + * @see StorageManager#getBatch(TupleSourceID, int, int) + */ + public TupleBatch getBatch(TupleSourceID storageID, int beginRow, String[] types) + throws TupleSourceNotFoundException, MetaMatrixComponentException { + + Map<Integer, TupleBatch> batches = storage.get(storageID); + + if(batches == null) { + throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", storageID)); //$NON-NLS-1$ + } + + synchronized(batches) { + TupleBatch batch = batches.get(beginRow); + if(batch == null) { + throw new MetaMatrixComponentException("unknown batch"); //$NON-NLS-1$ + } + return batch; + } + } + + /** + * @see StorageManager#removeStorageArea(TupleSourceID) + */ + public void removeBatches(TupleSourceID storageID) throws MetaMatrixComponentException { + storage.remove(storageID); + } + + /** + * @see StorageManager#shutdown() + */ + public void shutdown() { + this.storage.clear(); + this.storage = null; + } + + } + + private static BufferManager INSTANCE; + + /** + * Helper to get a buffer manager all set up for unmanaged standalone use. This is + * typically used for testing or when memory is not an issue. + * @return BufferManager ready for use + */ + public static BufferManager getStandaloneBufferManager() throws MetaMatrixComponentException { + if (INSTANCE == null) { + BufferManager bufferMgr = new BufferManagerImpl(); + Properties props = new Properties(); + props.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, String.valueOf(Long.MAX_VALUE)); + props.setProperty(BufferManagerPropertyNames.SESSION_USE_PERCENTAGE, "100"); //$NON-NLS-1$ + props.setProperty(BufferManagerPropertyNames.LOG_STATS_INTERVAL, "0"); //$NON-NLS-1$ + props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$ + bufferMgr.initialize("local", props); //$NON-NLS-1$ + + // Add unmanaged memory storage manager + bufferMgr.setStorageManager(new MemoryStorageManager()); + INSTANCE = bufferMgr; + } + + return INSTANCE; + } + +} Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBatchMap.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBatchMap.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBatchMap.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -28,6 +28,8 @@ import java.util.List; import java.util.Set; +import com.metamatrix.common.buffer.TupleSourceID; + import junit.framework.TestCase; @@ -35,14 +37,6 @@ * @since 4.3 */ public class TestBatchMap extends TestCase{ - private BatchMapValueTranslator batchMapValueTranslator = new BatchMapValueTranslator() { - public int getBeginRow(Object batchMapValue) { - return ((Batch)batchMapValue).getBeginRow(); - } - public int getEndRow(Object batchMapValue) { - return ((Batch)batchMapValue).getEndRow(); - } - }; public TestBatchMap(String arg0) { super(arg0); @@ -50,17 +44,16 @@ //no batch public void testAddAndGetBatch1() { - BatchMap batches = new BatchMap(batchMapValueTranslator); - assertTrue(batches.isEmpty()); + TupleSourceInfo batches = new TupleSourceInfo(null, null, null, null, null); assertNull(batches.getBatch(1)); assertNull(batches.getBatch(2)); } //one batch public void testAddAndGetBatch2() { - BatchMap batches = new BatchMap(batchMapValueTranslator); - Batch batch1 = new Batch(1,5); - batches.addBatch(1, batch1); + TupleSourceInfo batches = new TupleSourceInfo(null, null, null, null, null); + ManagedBatch batch1 = createManagedBatch(1, 5); + batches.addBatch(batch1); assertEquals(batch1, batches.getBatch(1)); assertEquals(batch1, batches.getBatch(2)); assertEquals(batch1, batches.getBatch(5)); @@ -70,11 +63,11 @@ //two batches public void testAddAndGetBatch3() { - BatchMap batches = new BatchMap(batchMapValueTranslator); - Batch batch1 = new Batch(1,5); - Batch batch2 = new Batch(6,10); - batches.addBatch(1, batch1); - batches.addBatch(6, batch2); + TupleSourceInfo batches = new TupleSourceInfo(null, null, null, null, null); + ManagedBatch batch1 = createManagedBatch(1, 5); + ManagedBatch batch2 = createManagedBatch(6, 10); + batches.addBatch(batch1); + batches.addBatch(batch2); assertEquals(batch1, batches.getBatch(1)); assertEquals(batch1, batches.getBatch(2)); assertEquals(batch1, batches.getBatch(5)); @@ -83,22 +76,26 @@ assertEquals(batch2, batches.getBatch(10)); assertNull(batches.getBatch(11)); } + + private ManagedBatch createManagedBatch(int begin, int end) { + return new ManagedBatch(new TupleSourceID("x"), begin, end, 0); //$NON-NLS-1$ + } //more batches, no adding order public void testAddAndGetBatch4() { - BatchMap batches = new BatchMap(batchMapValueTranslator); + TupleSourceInfo batches = new TupleSourceInfo(null, null, null, null, null); Set batchSet = new HashSet(); List batchList = new ArrayList(); for(int i=1; i<10000;) { - Batch batch = new Batch(i,i+4); + ManagedBatch batch = createManagedBatch(i, i + 4); batchSet.add(batch); batchList.add(batch); i += 5; } Iterator iter = batchSet.iterator(); while(iter.hasNext()) { - Batch next = (Batch)iter.next(); - batches.addBatch(next.getBeginRow(), next); + ManagedBatch next = (ManagedBatch)iter.next(); + batches.addBatch(next); } for(int i=1; i<10000;) { assertEquals(batchList.get(i/5), batches.getBatch(i)); @@ -112,39 +109,23 @@ } public void testAddAndRemoveBatch() { - BatchMap batches = new BatchMap(batchMapValueTranslator); - assertTrue(batches.isEmpty()); - Batch batch1 = new Batch(1,5); - Batch batch2 = new Batch(6,10); - batches.addBatch(1, batch1); - batches.addBatch(2, batch2); + TupleSourceInfo batches = new TupleSourceInfo(null, null, null, null, null); + ManagedBatch batch1 = createManagedBatch(1, 5); + ManagedBatch batch2 = createManagedBatch(6, 10); + batches.addBatch(batch1); + batches.addBatch(batch2); assertEquals(batch1, batches.getBatch(1)); batches.removeBatch(1); assertNull(batches.getBatch(1)); assertNull(batches.getBatch(2)); assertNull(batches.getBatch(5)); - batches.removeBatch(2); + batches.removeBatch(6); assertNull(batches.getBatch(1)); assertNull(batches.getBatch(2)); assertNull(batches.getBatch(5)); assertNull(batches.getBatch(6)); assertNull(batches.getBatch(8)); assertNull(batches.getBatch(10)); - assertTrue(batches.isEmpty()); } - class Batch{ - private int beginRow; - private int endRow; - Batch(int beginRow, int endRow){ - Batch.this.beginRow = beginRow; - Batch.this.endRow = endRow; - } - int getBeginRow() { - return Batch.this.beginRow; - } - int getEndRow() { - return Batch.this.endRow; - } - } } Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/TestBufferManagerImpl.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -47,7 +47,7 @@ import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; -import com.metamatrix.common.buffer.storage.memory.MemoryStorageManager; +import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager; import com.metamatrix.common.lob.ByteLobChunkStream; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.lob.LobChunkInputStream; @@ -69,7 +69,7 @@ super(arg0); } - public static BufferManager getTestBufferManager(long bytesAvailable, StorageManager sm1, StorageManager sm2) throws MetaMatrixComponentException { + public static BufferManager getTestBufferManager(long bytesAvailable, StorageManager sm2) throws MetaMatrixComponentException { // Get the properties for BufferManager Properties bmProps = new Properties(); bmProps.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, "" + bytesAvailable); //$NON-NLS-1$ @@ -77,30 +77,19 @@ BufferManager bufferManager = new BufferManagerImpl(); bufferManager.initialize("local", bmProps); //$NON-NLS-1$ - // Add storage managers - bufferManager.addStorageManager(sm1); - if(sm2 != null) { - bufferManager.addStorageManager(sm2); + bufferManager.setStorageManager(sm2); } return bufferManager; } - private StorageManager createMemoryStorageManager() { - return new MemoryStorageManager(); - } - private StorageManager createFakeDatabaseStorageManager() { - return new MemoryStorageManager() { - public int getStorageType() { - return StorageManager.TYPE_DATABASE; - } - }; + return new MemoryStorageManager(); } - public void helpTestAddBatches(StorageManager sm1, StorageManager sm2, int memorySize, int numBatches, int rowsPerBatch) throws MetaMatrixComponentException, MetaMatrixProcessingException { - BufferManager mgr = getTestBufferManager(memorySize, sm1, sm2); + public void helpTestAddBatches(StorageManager sm2, int memorySize, int numBatches, int rowsPerBatch) throws MetaMatrixComponentException, MetaMatrixProcessingException { + BufferManager mgr = getTestBufferManager(memorySize, sm2); List expectedRows = new ArrayList(); @@ -136,20 +125,28 @@ // Check batches in sm1 - 3 should fit in 1000 bytes of memory int batchesInMemory = (int) (memorySize * 1000000 / batchSize); - for(int b=0; b<batchesInMemory; b++) { - int begin = (b*rowsPerBatch) + 1; - - TupleBatch batch = sm1.getBatch(tsID, begin, null); - assertNotNull("batch " + (b+1) + " is null ", batch); //$NON-NLS-1$ //$NON-NLS-2$ - } + if (sm2 != null) { + for(int b=0; b<batchesInMemory; b++) { + int begin = (b*rowsPerBatch) + 1; + try { + sm2.getBatch(tsID, begin, null); + fail("Expected exception"); //$NON-NLS-1$ + } catch (TupleSourceNotFoundException e) { + + } catch (MetaMatrixComponentException e) { + + } + } + + // Check batches in sm2 + for(int b=batchesInMemory+1; b<numBatches; b++) { + int begin = (b*rowsPerBatch) + 1; + + TupleBatch batch = sm2.getBatch(tsID, begin, null); + assertNotNull("batch " + (b+1) + " is null ", batch); //$NON-NLS-1$ //$NON-NLS-2$ + } + } - // Check batches in sm2 - for(int b=batchesInMemory+1; b<numBatches; b++) { - int begin = (b*rowsPerBatch) + 1; - - TupleBatch batch = sm2.getBatch(tsID, begin, null); - assertNotNull("batch " + (b+1) + " is null ", batch); //$NON-NLS-1$ //$NON-NLS-2$ - } // Check row count int rowCount = mgr.getRowCount(tsID); @@ -168,16 +165,14 @@ } public void testSpanStorage() throws Exception { - helpTestAddBatches(createMemoryStorageManager(), - createFakeDatabaseStorageManager(), + helpTestAddBatches(createFakeDatabaseStorageManager(), 1, 50, 100); } public void testStandalone() throws Exception { - helpTestAddBatches(createMemoryStorageManager(), - null, + helpTestAddBatches(null, 10, 5, 10); @@ -218,7 +213,7 @@ } public void testCreateLobReference() throws Exception { - final BufferManagerImpl mgr = (BufferManagerImpl)getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + final BufferManagerImpl mgr = (BufferManagerImpl)getTestBufferManager(1, createFakeDatabaseStorageManager()); XMLType xml1 = new XMLType(new SQLXMLImpl("<foo/>")); //$NON-NLS-1$ XMLType xml2 = new XMLType(new SQLXMLImpl("<bar/>")); //$NON-NLS-1$ @@ -257,7 +252,7 @@ } public void testAddStreamablePart() throws Exception { - final BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + final BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); // save the lob List schema = new ArrayList(); @@ -318,7 +313,7 @@ public void testPinning1() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); List schema = new ArrayList(); schema.add("val"); //$NON-NLS-1$ @@ -349,7 +344,7 @@ } public void testUnpinOfUnpinnedBatch() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); List schema = new ArrayList(); schema.add("val"); //$NON-NLS-1$ @@ -403,7 +398,7 @@ } public void testDeadlockOnMultiThreadClean() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); List schema = new ArrayList(); schema.add("val"); //$NON-NLS-1$ @@ -440,7 +435,7 @@ } public void testSessionMax_Fail() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); TupleSourceID tsID = null; try { List schema = new ArrayList(); @@ -481,7 +476,7 @@ * @since 4.3 */ public void testDefect_18499() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); List schema = new ArrayList(); schema.add("col"); //$NON-NLS-1$ String[] schemaTypes = new String[] {DataTypeManager.DefaultDataTypes.STRING}; @@ -502,7 +497,7 @@ } public void testDefect18497() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); TupleSourceID tsID = null; try { List schema = new ArrayList(); @@ -536,7 +531,7 @@ //two threads do the cleaning at the same time public void testDefect19325() throws Exception{ - BufferManagerImpl mgr = (BufferManagerImpl)getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManagerImpl mgr = (BufferManagerImpl)getTestBufferManager(1, createFakeDatabaseStorageManager()); TupleSourceID tsID = null; List schema = new ArrayList(); schema.add("val"); //$NON-NLS-1$ @@ -572,7 +567,7 @@ //test many small batches public void testSmallBatches() throws Exception{ - BufferManager mgr = getTestBufferManager(50, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(50, createFakeDatabaseStorageManager()); TupleSourceID tsID = null; try { List schema = new ArrayList(); @@ -606,7 +601,7 @@ //going backward public void testPinning2() throws Exception { - BufferManager mgr = getTestBufferManager(1, createMemoryStorageManager(), createFakeDatabaseStorageManager()); + BufferManager mgr = getTestBufferManager(1, createFakeDatabaseStorageManager()); List schema = new ArrayList(); schema.add("val"); //$NON-NLS-1$ Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -322,7 +322,7 @@ } } - public void testAddTwice() { + public void testAddTwice() throws Exception { StorageManager sm = getStorageManager(null); TupleSourceID tsID = new TupleSourceID("local,1:0"); //$NON-NLS-1$ TupleBatch batch = exampleBatch(1, 20); @@ -331,14 +331,9 @@ // Add batch sm.addBatch(tsID, batch, null); - // Remove batch (does nothing) - sm.removeBatch(tsID, batch.getBeginRow()); - // Add batch again sm.addBatch(tsID, batch, null); - } catch(MetaMatrixException e) { - fail("Unexpected exception of type " + e.getClass().getName() + ": " + e.getMessage()); //$NON-NLS-1$ //$NON-NLS-2$ } finally { try { sm.removeBatches(tsID); @@ -527,7 +522,6 @@ // Remove if(added[batch]) { //System.out.println(tsID.toString() + ": removing batch " + batch); - sm.removeBatch(tsID, batches[batch].getBeginRow()); added[batch] = false; } } Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -38,8 +38,8 @@ import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerPropertyNames; import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager; import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.common.buffer.storage.memory.MemoryStorageManager; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.query.mapping.relational.QueryNode; import com.metamatrix.query.optimizer.TestOptimizer; @@ -491,17 +491,16 @@ private BufferManager createCustomBufferMgr(int batchSize) throws MetaMatrixComponentException { BufferManager bufferMgr = new BufferManagerImpl(); Properties props = new Properties(); - props.setProperty(BufferManagerPropertyNames.ID_CREATOR, "com.metamatrix.common.buffer.impl.LongIDCreator"); //$NON-NLS-1$ - props.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, "" + Long.MAX_VALUE); //$NON-NLS-1$ + props.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, String.valueOf(Long.MAX_VALUE)); props.setProperty(BufferManagerPropertyNames.SESSION_USE_PERCENTAGE, "100"); //$NON-NLS-1$ props.setProperty(BufferManagerPropertyNames.LOG_STATS_INTERVAL, "0"); //$NON-NLS-1$ props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$ - props.setProperty(BufferManagerPropertyNames.PROCESSOR_BATCH_SIZE, "" + batchSize); //$NON-NLS-1$ - props.setProperty(BufferManagerPropertyNames.CONNECTOR_BATCH_SIZE, "" + batchSize);//$NON-NLS-1$ + props.setProperty(BufferManagerPropertyNames.PROCESSOR_BATCH_SIZE, String.valueOf(batchSize)); + props.setProperty(BufferManagerPropertyNames.CONNECTOR_BATCH_SIZE, String.valueOf(batchSize)); bufferMgr.initialize("local", props); //$NON-NLS-1$ // Add unmanaged memory storage manager - bufferMgr.addStorageManager(new MemoryStorageManager()); + bufferMgr.setStorageManager(new MemoryStorageManager()); return bufferMgr; } Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -33,8 +33,8 @@ import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager; import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.common.buffer.storage.memory.MemoryStorageManager; import com.metamatrix.core.MetaMatrixRuntimeException; @@ -45,7 +45,6 @@ static BufferManager getTestBufferManager(long bytesAvailable) { // Get the properties for BufferManager Properties bmProps = new Properties(); - bmProps.setProperty(BufferManagerPropertyNames.ID_CREATOR, "com.metamatrix.common.buffer.impl.LongIDCreator"); //$NON-NLS-1$ bmProps.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, "" + bytesAvailable); //$NON-NLS-1$ return createBufferManager(bmProps); } @@ -54,7 +53,6 @@ // Get the properties for BufferManager Properties bmProps = new Properties(); - bmProps.setProperty(BufferManagerPropertyNames.ID_CREATOR, "com.metamatrix.common.buffer.impl.LongIDCreator"); //$NON-NLS-1$ bmProps.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, "" + bytesAvailable); //$NON-NLS-1$ bmProps.setProperty(BufferManagerPropertyNames.PROCESSOR_BATCH_SIZE, "" + procBatchSize); //$NON-NLS-1$ bmProps.setProperty(BufferManagerPropertyNames.CONNECTOR_BATCH_SIZE, "" + connectorBatchSize); //$NON-NLS-1$ @@ -65,7 +63,6 @@ // Get the properties for BufferManager Properties bmProps = new Properties(); - bmProps.setProperty(BufferManagerPropertyNames.ID_CREATOR, "com.metamatrix.common.buffer.impl.LongIDCreator"); //$NON-NLS-1$ bmProps.setProperty(BufferManagerPropertyNames.MEMORY_AVAILABLE, "" + bytesAvailable); //$NON-NLS-1$ bmProps.setProperty(BufferManagerPropertyNames.PROCESSOR_BATCH_SIZE, "" + procBatchSize); //$NON-NLS-1$ return createBufferManager(bmProps); @@ -81,22 +78,14 @@ } // Add storage managers - bufferManager.addStorageManager(createMemoryStorageManager()); - bufferManager.addStorageManager(createFakeDatabaseStorageManager()); + bufferManager.setStorageManager(createFakeDatabaseStorageManager()); return bufferManager; } - private static StorageManager createMemoryStorageManager() { - return new MemoryStorageManager(); - } private static StorageManager createFakeDatabaseStorageManager() { - return new MemoryStorageManager() { - public int getStorageType() { - return StorageManager.TYPE_DATABASE; - } - }; + return new MemoryStorageManager(); } public static class TestableBufferManagerImpl extends BufferManagerImpl { Modified: trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java =================================================================== --- trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -38,7 +38,6 @@ import com.metamatrix.common.buffer.StorageManager; import com.metamatrix.common.buffer.impl.BufferManagerImpl; import com.metamatrix.common.buffer.storage.file.FileStorageManager; -import com.metamatrix.common.buffer.storage.memory.MemoryStorageManager; import com.metamatrix.dqp.embedded.DQPEmbeddedPlugin; import com.metamatrix.dqp.service.BufferService; import com.metamatrix.dqp.service.ConfigurationService; @@ -102,7 +101,6 @@ // Set up buffer configuration properties Properties bufferProps = new Properties(); - bufferProps.setProperty(BufferManagerPropertyNames.ID_CREATOR, DEFAULT_ID_CREATOR); bufferProps.setProperty(BufferManagerPropertyNames.SESSION_USE_PERCENTAGE, DEFAULT_SESSION_USE_PERCENTAGE); bufferProps.setProperty(BufferManagerPropertyNames.LOG_STATS_INTERVAL, DEFAULT_LOG_STATS_INTERVAL); bufferProps.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, DEFAULT_MANAGEMENT_INTERVAL); @@ -126,16 +124,13 @@ fsmProps.setProperty(BufferManagerPropertyNames.MAX_OPEN_FILES, DEFAULT_MAX_OPEN_FILES); StorageManager fsm = new FileStorageManager(); fsm.initialize(fsmProps); - this.bufferMgr.addStorageManager(fsm); + this.bufferMgr.setStorageManager(fsm); // start the file storrage manager in clean state // wise FileStorageManager is smart enough to clen up after itself cleanDirectory(bufferDir); } - // Add unmanaged memory storage manager - this.bufferMgr.addStorageManager(new MemoryStorageManager()); - } catch(MetaMatrixComponentException e) { throw new ApplicationLifecycleException(e, DQPEmbeddedPlugin.Util.getString("LocalBufferService.Failed_initializing_buffer_manager._8")); //$NON-NLS-1$ } catch(IOException e) { Modified: trunk/server/src/main/java/com/metamatrix/server/dqp/service/PlatformBufferService.java =================================================================== --- trunk/server/src/main/java/com/metamatrix/server/dqp/service/PlatformBufferService.java 2009-08-11 01:26:05 UTC (rev 1232) +++ trunk/server/src/main/java/com/metamatrix/server/dqp/service/PlatformBufferService.java 2009-08-11 03:25:47 UTC (rev 1233) @@ -33,8 +33,10 @@ import com.metamatrix.common.application.exception.ApplicationInitializationException; import com.metamatrix.common.application.exception.ApplicationLifecycleException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.BufferManagerFactory; import com.metamatrix.common.buffer.BufferManagerPropertyNames; +import com.metamatrix.common.buffer.StorageManager; +import com.metamatrix.common.buffer.impl.BufferManagerImpl; +import com.metamatrix.common.buffer.storage.file.FileStorageManager; import com.metamatrix.common.config.api.Host; import com.metamatrix.common.util.PropertiesUtils; import com.metamatrix.core.util.FileUtils; @@ -87,12 +89,33 @@ props.setProperty(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY, dir); try { - bufferMgr = BufferManagerFactory.getServerBufferManager(host.getFullName()+"-"+processName, props); //$NON-NLS-1$ + bufferMgr = getServerBufferManager(host.getFullName()+"-"+processName, props); //$NON-NLS-1$ } catch (MetaMatrixComponentException e) { throw new ApplicationLifecycleException(e); } } + + /** + * Helper to get a buffer manager all set up for unmanaged standalone use. This is + * typically used for testing or when memory is not an issue. + * @param lookup Lookup implementation to use + * @param props Configuration properties + * @return BufferManager ready for use + */ + public static BufferManager getServerBufferManager(String lookup, Properties props) throws MetaMatrixComponentException { + Properties bmProps = PropertiesUtils.clone(props, false); + // Construct buffer manager + BufferManager bufferManager = new BufferManagerImpl(); + bufferManager.initialize(lookup, bmProps); + // Get the properties for FileStorageManager and create. + StorageManager fsm = new FileStorageManager(); + fsm.initialize(bmProps); + bufferManager.setStorageManager(fsm); + + return bufferManager; + } + /* * @see com.metamatrix.common.application.ApplicationService#stop() */

16 years, 8 months

1
0
0 / 0

teiid SVN: r1232 - trunk/connectors/sandbox/connector-oracle-spatial.

by teiid-commits＠lists.jboss.org

Author: shawkins Date: 2009-08-10 21:26:05 -0400 (Mon, 10 Aug 2009) New Revision: 1232 Modified: trunk/connectors/sandbox/connector-oracle-spatial/pom.xml Log: TEIID-751 changing to use type test-jar Modified: trunk/connectors/sandbox/connector-oracle-spatial/pom.xml =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/pom.xml 2009-08-10 21:03:49 UTC (rev 1231) +++ trunk/connectors/sandbox/connector-oracle-spatial/pom.xml 2009-08-11 01:26:05 UTC (rev 1232) @@ -1,39 +1,40 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <parent> - <artifactId>sandbox</artifactId> - <groupId>org.jboss.teiid.connectors</groupId> - <version>6.2.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>connector-oracle-spatial</artifactId> - <groupId>org.jboss.teiid.connectors.sandbox</groupId> - <name>Oracle Spatial Connector</name> - <description>Oracle Spatial Connector</description> - <dependencies> - <dependency> - <groupId>org.jboss.teiid</groupId> - <artifactId>teiid-connector-api</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.jboss.teiid</groupId> - <artifactId>teiid-common-core</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.jboss.teiid.connectors</groupId> - <artifactId>connector-jdbc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.jboss.teiid.connectors</groupId> - <artifactId>connector-jdbc</artifactId> - <version>${project.version}</version> - <type>jar</type> - <classifier>tests</classifier> - <scope>test</scope> - </dependency> - </dependencies> +<project + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <parent> + <artifactId>sandbox</artifactId> + <groupId>org.jboss.teiid.connectors</groupId> + <version>6.2.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>connector-oracle-spatial</artifactId> + <groupId>org.jboss.teiid.connectors.sandbox</groupId> + <name>Oracle Spatial Connector</name> + <description>Oracle Spatial Connector</description> + <dependencies> + <dependency> + <groupId>org.jboss.teiid</groupId> + <artifactId>teiid-connector-api</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.jboss.teiid</groupId> + <artifactId>teiid-common-core</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.jboss.teiid.connectors</groupId> + <artifactId>connector-jdbc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.jboss.teiid.connectors</groupId> + <artifactId>connector-jdbc</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> </project> \ No newline at end of file

16 years, 8 months

1
0
0 / 0

teiid SVN: r1231 - in trunk/connectors/sandbox/connector-oracle-spatial: src/main/java/com/metamatrix/connector/jdbc/oracle/spatial and 7 other directories.

by teiid-commits＠lists.jboss.org

Author: loleary Date: 2009-08-10 17:03:49 -0400 (Mon, 10 Aug 2009) New Revision: 1231 Added: trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/spatial/ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/spatial/TestOracleSpatialTranslator.java Modified: trunk/connectors/sandbox/connector-oracle-spatial/pom.xml trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialFunctionModifier.java Log: TEIID-751: SDO_WITHIN_DISTANCE function of Oracle Spatial Connector is pushing string literal to data source Changed OracleSpatialFunctionModifier.addParamWithConversion(List<Object> objs, IExpression expression) : void to remove string escape from SQL if expression is ILiteral with a value who's type is String. Added unit tests to give rewriter some basic coverage. Reviewed By: lliang Modified: trunk/connectors/sandbox/connector-oracle-spatial/pom.xml =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/pom.xml 2009-08-10 20:48:21 UTC (rev 1230) +++ trunk/connectors/sandbox/connector-oracle-spatial/pom.xml 2009-08-10 21:03:49 UTC (rev 1231) @@ -27,5 +27,13 @@ <artifactId>connector-jdbc</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.jboss.teiid.connectors</groupId> + <artifactId>connector-jdbc</artifactId> + <version>${project.version}</version> + <type>jar</type> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> </dependencies> </project> \ No newline at end of file Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialFunctionModifier.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialFunctionModifier.java 2009-08-10 20:48:21 UTC (rev 1230) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialFunctionModifier.java 2009-08-10 21:03:49 UTC (rev 1231) @@ -31,11 +31,14 @@ public class OracleSpatialFunctionModifier extends BasicFunctionModifier { - protected void addParamWithConversion(List objs, + protected void addParamWithConversion(List<Object> objs, IExpression expression) { - if ((expression instanceof ILiteral) && (((ILiteral)expression).getType() == String.class)) - objs.add(((String)((ILiteral)expression).getValue())); - else - objs.add(expression); + if ((expression instanceof ILiteral) + && (((ILiteral) expression).getValue() != null) + && (((ILiteral) expression).getValue() instanceof String)) + objs.add(((String) ((ILiteral) expression).getValue())); + else + objs.add(expression); } + } \ No newline at end of file Added: trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/spatial/TestOracleSpatialTranslator.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/spatial/TestOracleSpatialTranslator.java (rev 0) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/spatial/TestOracleSpatialTranslator.java 2009-08-10 21:03:49 UTC (rev 1231) @@ -0,0 +1,457 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.connector.jdbc.oracle.spatial; + + +import java.io.File; +import java.net.URL; +import java.util.Properties; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.teiid.connector.api.ConnectorException; +import org.teiid.connector.jdbc.MetadataFactory; +import org.teiid.connector.jdbc.translator.TranslatedCommand; +import org.teiid.connector.jdbc.translator.Translator; +import org.teiid.connector.language.ICommand; + +import com.metamatrix.cdk.api.EnvironmentUtility; +import com.metamatrix.cdk.unittest.FakeTranslationFactory; +import com.metamatrix.connector.jdbc.oracle.spatial.OracleSpatialSQLTranslator; +import com.metamatrix.core.MetaMatrixRuntimeException; +import com.metamatrix.query.function.FunctionLibraryManager; +import com.metamatrix.query.function.UDFSource; + +public class TestOracleSpatialTranslator { + + /** + * An instance of {@link Translator} which has already been initialized. + */ + private static Translator TRANSLATOR; + + /** + * Performs setup tasks that should be executed prior to an instance of this + * class being created. This method should only be executed once and does + * not protect from multiple executions. It is intended to be executed by + * the JUnit4 test framework. + * + * This method sets {@link TestOracleTranslator#TRANSLATOR} to an instance + * of {@link OracleSpatialSQLTranslator} and then calls its {@link OracleSpatialSQLTranslator#initialize(ConnectorEnvironment)} + * method. + * + * This method loads the function definitions supported by the Oracle Spatial + * Connector into the global instance of <code>FunctionLibraryManager</code> + * so that they are resolvable during query parsing. + * + * @throws Exception + */ + @BeforeClass public static void oneTimeSetup() throws Exception { + TRANSLATOR = new OracleSpatialSQLTranslator(); + TRANSLATOR.initialize(EnvironmentUtility.createEnvironment(new Properties(), false)); + // Define a UDFSource to hold the reference to our function definitions + File udfFile = new File("src/main/resources/OracleSpatialFunctions.xmi"); //$NON-NLS-1$; + URL[] urls = new URL[0]; + UDFSource udfSource = new UDFSource(udfFile.toURI().toURL(), urls); + FunctionLibraryManager.deregisterSource(udfSource); + FunctionLibraryManager.registerSource(udfSource); + } + + /** + * Performs cleanup tasks that should be executed after all test methods + * have been executed. This method should only be executed once and does + * not protect from multiple executions. It is intended to be executed by + * the JUnit4 test framework. + * + * This method unloads the function definitions supported by the Oracle Spatial + * Connector from the global instance of <code>FunctionLibraryManager</code> + * so that they are no longer resolvable during query parsing. + * + * @throws Exception + */ + @AfterClass public static void oneTimeFinished() throws Exception { + // Define a UDFSource to hold the reference to our function definitions + File udfFile = new File("src/main/resources/OracleSpatialFunctions.xmi"); //$NON-NLS-1$; + URL[] urls = new URL[0]; + UDFSource udfSource = new UDFSource(udfFile.toURI().toURL(), urls); + FunctionLibraryManager.deregisterSource(udfSource); + } + + private void helpTestVisitor(String input, String expectedOutput) throws ConnectorException { + // Convert from sql to objects + ICommand obj = FakeTranslationFactory.getInstance().getAutoIncrementTranslationUtility().parseCommand(input); + + TranslatedCommand tc = new TranslatedCommand(EnvironmentUtility.createSecurityContext("user"), TRANSLATOR); //$NON-NLS-1$ + tc.translateCommand(obj); + + // Check stuff + org.junit.Assert.assertEquals("Did not get correct sql", expectedOutput, tc.getSql()); //$NON-NLS-1$ + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_relate(Object element, Object element, String literal) in its + * criteria into a source specific command. + * + * @throws Exception + */ + @Ignore("TEIID-757: Translator needs fixing so that spacing is valid between hint and query") + @Test public void testRewrite_sdo_relate() throws Exception { + // TODO TEIID-757: Translator needs fixing so that spacing is valid between hint and query + String input = "SELECT a.INTKEY FROM BQT1.SMALLA A, BQT1.SMALLB B WHERE sdo_relate(A.OBJECTVALUE, b.OBJECTVALUE, 'mask=ANYINTERACT') = true"; //$NON-NLS-1$ + String output = "SELECT /*+ ORDERED */ A.IntKey FROM SmallA A, SmallB B WHERE SDO_RELATE(A.ObjectValue, B.ObjectValue, 'mask=ANYINTERACT') = 'true'"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_within_distance(Object element, String literal, String literal) + * in its criteria into a source specific command. + * + * @throws Exception + */ + @Test public void testRewrite_sdo_within_distance() throws Exception { + String input = "SELECT INTKEY FROM BQT1.SMALLA WHERE sdo_within_distance(OBJECTVALUE, 'SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL)', 'DISTANCE=25.0 UNIT=NAUT_MILE') = true"; //$NON-NLS-1$ + String output = "SELECT SmallA.IntKey FROM SmallA WHERE SDO_WITHIN_DISTANCE(SmallA.ObjectValue, SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL), 'DISTANCE=25.0 UNIT=NAUT_MILE') = 'true'"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_within_distance(String literal, Object element, String literal) + * in its criteria into a source specific command. + * + * @throws Exception + */ + @Test public void testRewrite_sdo_within_distance2() throws Exception { + String input = "SELECT INTKEY FROM BQT1.SMALLA WHERE sdo_within_distance('SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL)', OBJECTVALUE, 'DISTANCE=25.0 UNIT=NAUT_MILE') = true"; //$NON-NLS-1$ + String output = "SELECT SmallA.IntKey FROM SmallA WHERE SDO_WITHIN_DISTANCE(SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL), SmallA.ObjectValue, 'DISTANCE=25.0 UNIT=NAUT_MILE') = 'true'"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_within_distance(String element, String literal, String literal) + * in its criteria into a source specific command. + * + * @throws Exception + */ + @Test public void testRewrite_sdo_within_distance3() throws Exception { + String input = "SELECT INTKEY FROM BQT1.SMALLA WHERE sdo_within_distance(STRINGKEY, 'SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL)', 'DISTANCE=25.0 UNIT=NAUT_MILE') = true"; //$NON-NLS-1$ + // using ? for bind value as rewriter marks the criteria as bindEligible + // due to literal of type Object appearing in left side of criteria. + // The literal Object is a result of the sdo_within_distance function + // signature being sdo_within_distance(string, object, string) : string + // as the signature was the best match for this query. + String output = "SELECT SmallA.IntKey FROM SmallA WHERE SDO_WITHIN_DISTANCE(SmallA.StringKey, SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL), 'DISTANCE=25.0 UNIT=NAUT_MILE') = ?"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_within_distance(String literal, String literal, String literal) + * in its criteria into a source specific command. + * + * @throws Exception + */ + @Test public void testRewrite_sdo_within_distance4() throws Exception { + String input = "SELECT INTKEY FROM BQT1.SMALLA WHERE sdo_within_distance('SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL)', 'SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL)', 'DISTANCE=25.0 UNIT=NAUT_MILE') = true"; //$NON-NLS-1$ + // using ? for bind value as rewriter marks the criteria as bindEligible + // due to literal of type Object appearing in left side of criteria. + // The literal Object is a result of the sdo_within_distance function + // signature being sdo_within_distance(string, object, string) : string + // as the signature was the best match for this query. + String output = "SELECT SmallA.IntKey FROM SmallA WHERE SDO_WITHIN_DISTANCE(SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL), SDO_GEOMETRY(2001, 8307, MDSYS.SDO_POINT_TYPE(90.0, -45.0, NULL), NULL, NULL), 'DISTANCE=25.0 UNIT=NAUT_MILE') = ?"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_within_distance(Object element, Object element, String literal) + * in its criteria into a source specific command. + * + * @throws Exception + */ + @Test public void testRewrite_sdo_within_distance5() throws Exception { + String input = "SELECT a.INTKEY FROM BQT1.SMALLA A, BQT1.SMALLB B WHERE sdo_within_distance(a.OBJECTVALUE, b.OBJECTVALUE, 'DISTANCE=25.0 UNIT=NAUT_MILE') = true"; //$NON-NLS-1$ + String output = "SELECT A.IntKey FROM SmallA A, SmallB B WHERE SDO_WITHIN_DISTANCE(A.ObjectValue, B.ObjectValue, 'DISTANCE=25.0 UNIT=NAUT_MILE') = 'true'"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + /** + * Test query rewriter's ability to translate a query which uses + * <code>sdo_within_distance(Object element, String literal) in its criteria + * into a source specific command. + * + * This use case should result in an exception being thrown by the query + * resolver as the function signature does not exist for only two parameters. + * + * @throws Exception + */ + @Test(expected=MetaMatrixRuntimeException.class) public void testRewrite_sdo_within_distance_wrongNumParams() throws Exception { + String input = "SELECT INTKEY FROM BQT1.SMALLA WHERE sdo_within_distance(OBJECTVALUE, 'DISTANCE=25.0 UNIT=NAUT_MILE') = true"; //$NON-NLS-1$ + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, null, + TRANSLATOR); + } + + @Test public void testInsertWithSequnce() throws Exception { + helpTestVisitor("insert into test.group (e0) values (1)", "INSERT INTO group (e0, e1) VALUES (1, MYSEQUENCE.nextVal)"); //$NON-NLS-1$ //$NON-NLS-2$ + } + + @Test public void testInsertWithSequnce1() throws Exception { + helpTestVisitor("insert into test.group (e0, e1) values (1, 'x')", "INSERT INTO group (e0, e1) VALUES (1, 'x')"); //$NON-NLS-1$ //$NON-NLS-2$ + } + + @Test public void testJoins() throws Exception { + String input = "select smalla.intkey from bqt1.smalla inner join bqt1.smallb on smalla.stringkey=smallb.stringkey cross join bqt1.mediuma"; //$NON-NLS-1$ + String output = "SELECT SmallA.IntKey FROM SmallA INNER JOIN SmallB ON SmallA.StringKey = SmallB.StringKey CROSS JOIN MediumA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, + output, TRANSLATOR); + } + + @Test public void testJoins2() throws Exception { + String input = "select smalla.intkey from bqt1.smalla cross join (bqt1.smallb cross join bqt1.mediuma)"; //$NON-NLS-1$ + String output = "SELECT SmallA.IntKey FROM SmallA CROSS JOIN (SmallB CROSS JOIN MediumA)"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, + output, TRANSLATOR); + } + + @Test public void testRewriteConversion1() throws Exception { + String input = "SELECT char(convert(STRINGNUM, integer) + 100) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT chr((to_number(SmallA.StringNum) + 100)) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testRewriteConversion2() throws Exception { + String input = "SELECT convert(STRINGNUM, long) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_number(SmallA.StringNum) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testRewriteConversion3() throws Exception { + String input = "SELECT convert(convert(STRINGNUM, long), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(to_number(SmallA.StringNum)) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testRewriteConversion4() throws Exception { + String input = "SELECT convert(convert(TIMESTAMPVALUE, date), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(trunc(SmallA.TimestampValue), 'YYYY-MM-DD') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion5() throws Exception { + String input = "SELECT convert(convert(TIMESTAMPVALUE, time), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(to_date(('1970-01-01 ' || to_char(SmallA.TimestampValue, 'HH24:MI:SS')), 'YYYY-MM-DD HH24:MI:SS'), 'HH24:MI:SS') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion6() throws Exception { + String input = "SELECT convert(convert(TIMEVALUE, timestamp), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(cast(SmallA.TimeValue AS timestamp), 'YYYY-MM-DD HH24:MI:SS.FF') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion8() throws Exception { + String input = "SELECT nvl(INTNUM, 'otherString') FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT nvl(to_char(SmallA.IntNum), 'otherString') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion7() throws Exception { + String input = "SELECT convert(convert(STRINGNUM, integer), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(to_number(SmallA.StringNum)) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Ignore("TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is") + @Test public void testRewriteLocate() throws Exception { + // TODO TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is + String input = "SELECT locate(INTNUM, 'chimp', 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT instr('chimp', to_char(SmallA.IntNum), 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteLocate2() throws Exception { + String input = "SELECT locate(STRINGNUM, 'chimp') FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT instr('chimp', SmallA.StringNum) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Ignore("TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is") + @Test public void testRewriteLocate3() throws Exception { + // TODO TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is + String input = "SELECT locate(INTNUM, '234567890', 1) FROM BQT1.SMALLA WHERE INTKEY = 26"; //$NON-NLS-1$ + String output = "SELECT instr('234567890', to_char(SmallA.IntNum), 1) FROM SmallA WHERE SmallA.IntKey = 26"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteLocate4() throws Exception { + String input = "SELECT locate('c', 'chimp', 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT 1 FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Ignore("TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is 1 if a value of < 1 is given") + @Test public void testRewriteLocate5() throws Exception { + // TODO TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is 1 if a value of < 1 is given + String input = "SELECT locate(STRINGNUM, 'chimp', -5) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT instr('chimp', SmallA.StringNum, 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteSubstring1() throws Exception { + String input = "SELECT substring(StringNum, 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT substr(SmallA.StringNum, 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteSubstring2() throws Exception { + String input = "SELECT substring(StringNum, 1, 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT substr(SmallA.StringNum, 1, 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteUnionWithOrderBy() throws Exception { + String input = "SELECT IntKey FROM BQT1.SMALLA UNION SELECT IntKey FROM BQT1.SMALLB ORDER BY IntKey"; //$NON-NLS-1$ + String output = "SELECT SmallA.IntKey FROM SmallA UNION SELECT SmallB.IntKey FROM SmallB ORDER BY IntKey NULLS FIRST"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit1() throws Exception { + String input = "select intkey from bqt1.smalla limit 10, 0"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT VIEW_FOR_LIMIT.*, ROWNUM ROWNUM_ FROM (SELECT SmallA.IntKey FROM SmallA) VIEW_FOR_LIMIT WHERE ROWNUM <= 10) WHERE ROWNUM_ > 10"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit2() throws Exception { + String input = "select intkey from bqt1.smalla limit 0, 10"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT SmallA.IntKey FROM SmallA) WHERE ROWNUM <= 10"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit3() throws Exception { + String input = "select intkey from bqt1.smalla limit 1, 10"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT VIEW_FOR_LIMIT.*, ROWNUM ROWNUM_ FROM (SELECT SmallA.IntKey FROM SmallA) VIEW_FOR_LIMIT WHERE ROWNUM <= 11) WHERE ROWNUM_ > 1"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit4() throws Exception { + String input = "select intkey from bqt1.mediuma limit 100"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT MediumA.IntKey FROM MediumA) WHERE ROWNUM <= 100"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit5() throws Exception { + String input = "select intkey from bqt1.mediuma limit 50, 100"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT VIEW_FOR_LIMIT.*, ROWNUM ROWNUM_ FROM (SELECT MediumA.IntKey FROM MediumA) VIEW_FOR_LIMIT WHERE ROWNUM <= 150) WHERE ROWNUM_ > 50"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testConcat2_useLiteral() throws Exception { + String input = "select concat2(stringnum,'_xx') from bqt1.Smalla"; //$NON-NLS-1$ + String output = "SELECT concat(nvl(SmallA.StringNum, ''), '_xx') FROM SmallA"; //$NON-NLS-1$ + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testConcat2() throws Exception { + String input = "select concat2(stringnum, stringkey) from bqt1.Smalla"; //$NON-NLS-1$ + String output = "SELECT CASE WHEN (SmallA.StringNum IS NULL) AND (SmallA.StringKey IS NULL) THEN NULL ELSE concat(nvl(SmallA.StringNum, ''), nvl(SmallA.StringKey, '')) END FROM SmallA"; //$NON-NLS-1$ + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + +} Property changes on: trunk/connectors/sandbox/connector-oracle-spatial/src/test/java/org/teiid/connector/jdbc/oracle/spatial/TestOracleSpatialTranslator.java ___________________________________________________________________ Name: svn:mime-type + text/plain

16 years, 8 months

1
0
0 / 0

teiid SVN: r1230 - trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial.

by teiid-commits＠lists.jboss.org

Author: loleary Date: 2009-08-10 16:48:21 -0400 (Mon, 10 Aug 2009) New Revision: 1230 Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/FilterFunctionModifier.java trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/NearestNeighborFunctionModifier.java trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/RelateFunctionModifier.java trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/WithinDistanceFunctionModifier.java Log: Added type to List instances to prevent compile warnings. Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/FilterFunctionModifier.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/FilterFunctionModifier.java 2009-08-10 20:45:59 UTC (rev 1229) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/FilterFunctionModifier.java 2009-08-10 20:48:21 UTC (rev 1230) @@ -42,7 +42,7 @@ * marks */ public List translate(IFunction function) { - List objs = new ArrayList(); + List<Object> objs = new ArrayList<Object>(); objs.add("SDO_FILTER"); //$NON-NLS-1$ objs.add("("); //$NON-NLS-1$ List<IExpression> params = function.getParameters(); Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/NearestNeighborFunctionModifier.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/NearestNeighborFunctionModifier.java 2009-08-10 20:45:59 UTC (rev 1229) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/NearestNeighborFunctionModifier.java 2009-08-10 20:48:21 UTC (rev 1230) @@ -42,7 +42,7 @@ * marks */ public List translate(IFunction function) { - List objs = new ArrayList(); + List<Object> objs = new ArrayList<Object>(); objs.add("SDO_NN"); // recast name from sdoNN to SDO_NN //$NON-NLS-1$ objs.add("("); //$NON-NLS-1$ List<IExpression> params = function.getParameters(); Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/RelateFunctionModifier.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/RelateFunctionModifier.java 2009-08-10 20:45:59 UTC (rev 1229) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/RelateFunctionModifier.java 2009-08-10 20:48:21 UTC (rev 1230) @@ -44,7 +44,7 @@ * marks */ public List translate( IFunction function ) { - List objs = new ArrayList(); + List<Object> objs = new ArrayList<Object>(); objs.add("SDO_RELATE"); // recast name from sdoRelate to SDO_RELATE //$NON-NLS-1$ objs.add("("); //$NON-NLS-1$ List<IExpression> params = function.getParameters(); Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/WithinDistanceFunctionModifier.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/WithinDistanceFunctionModifier.java 2009-08-10 20:45:59 UTC (rev 1229) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/WithinDistanceFunctionModifier.java 2009-08-10 20:48:21 UTC (rev 1230) @@ -42,7 +42,7 @@ * marks */ public List translate(IFunction function) { - List objs = new ArrayList(); + List<Object> objs = new ArrayList<Object>(); objs.add("SDO_WITHIN_DISTANCE"); // recast name from sdoNN to SDO_NN //$NON-NLS-1$ objs.add("("); //$NON-NLS-1$ List<IExpression> params = function.getParameters();

16 years, 8 months

1
0
0 / 0

teiid SVN: r1229 - trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial.

by teiid-commits＠lists.jboss.org

Author: loleary Date: 2009-08-10 16:45:59 -0400 (Mon, 10 Aug 2009) New Revision: 1229 Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialSQLTranslator.java Log: TEIID-766: Added call to super.initialize so that function modifiers from OracleSQLTranslator could be registered. Also added super call in modifyCommand to offer OracleSQLTranslator specific modifications. Modified: trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialSQLTranslator.java =================================================================== --- trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialSQLTranslator.java 2009-08-10 20:36:00 UTC (rev 1228) +++ trunk/connectors/sandbox/connector-oracle-spatial/src/main/java/com/metamatrix/connector/jdbc/oracle/spatial/OracleSpatialSQLTranslator.java 2009-08-10 20:45:59 UTC (rev 1229) @@ -45,6 +45,7 @@ @Override public void initialize(ConnectorEnvironment env) throws ConnectorException { + super.initialize(env); Iterator<String> iter = OracleSpatialFunctions.relateFunctions.iterator(); while (iter.hasNext()) { registerFunctionModifier(iter.next(), new RelateFunctionModifier()); @@ -102,7 +103,7 @@ } } - return command; + return super.modifyCommand(command, context); } /**

16 years, 8 months

1
0
0 / 0

teiid SVN: r1228 - trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/oracle.

by teiid-commits＠lists.jboss.org

Author: loleary Date: 2009-08-10 16:36:00 -0400 (Mon, 10 Aug 2009) New Revision: 1228 Modified: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/oracle/TestOracleTranslator.java Log: Converted to use JUnit4 framework Added test methods from R050504c branch Modified: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/oracle/TestOracleTranslator.java =================================================================== --- trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/oracle/TestOracleTranslator.java 2009-08-10 18:26:58 UTC (rev 1227) +++ trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/oracle/TestOracleTranslator.java 2009-08-10 20:36:00 UTC (rev 1228) @@ -24,8 +24,9 @@ import java.util.Properties; -import junit.framework.TestCase; - +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; import org.teiid.connector.api.ConnectorException; import org.teiid.connector.jdbc.MetadataFactory; import org.teiid.connector.jdbc.translator.TranslatedCommand; @@ -35,39 +36,49 @@ import com.metamatrix.cdk.api.EnvironmentUtility; import com.metamatrix.cdk.unittest.FakeTranslationFactory; -public class TestOracleTranslator extends TestCase { +public class TestOracleTranslator { - private static Translator TRANSLATOR; + /** + * An instance of {@link Translator} which has already been initialized. + */ + private static Translator TRANSLATOR; - static { - try { - TRANSLATOR = new OracleSQLTranslator(); - TRANSLATOR.initialize(EnvironmentUtility.createEnvironment(new Properties(), false)); - } catch(ConnectorException e) { - e.printStackTrace(); - } + /** + * Performs setup tasks that should be executed prior to an instance of this + * class being created. This method should only be executed once and does + * not protect from multiple executions. It is intended to be executed by + * the JUnit4 test framework. + * + * This method sets {@link TestOracleTranslator#TRANSLATOR} to an instance + * of {@link OracleSQLTranslator} and then calls its {@link OracleSQLTranslator#initialize(ConnectorEnvironment)} + * method. + * @throws Exception + */ + @BeforeClass public static void oneTimeSetup() throws Exception { + TRANSLATOR = new OracleSQLTranslator(); + TRANSLATOR.initialize(EnvironmentUtility.createEnvironment(new Properties(), false)); } - public void helpTestVisitor(String input, String expectedOutput) throws ConnectorException { + private void helpTestVisitor(String input, String expectedOutput) throws ConnectorException { // Convert from sql to objects ICommand obj = FakeTranslationFactory.getInstance().getAutoIncrementTranslationUtility().parseCommand(input); - TranslatedCommand tc = new TranslatedCommand(EnvironmentUtility.createSecurityContext("user"), TRANSLATOR); + TranslatedCommand tc = new TranslatedCommand(EnvironmentUtility.createSecurityContext("user"), TRANSLATOR); //$NON-NLS-1$ tc.translateCommand(obj); // Check stuff - assertEquals("Did not get correct sql", expectedOutput, tc.getSql()); //$NON-NLS-1$ + org.junit.Assert.assertEquals("Did not get correct sql", expectedOutput, tc.getSql()); //$NON-NLS-1$ } - public void testInsertWithSequnce() throws Exception { - helpTestVisitor("insert into test.group (e0) values (1)", "INSERT INTO group (e0, e1) VALUES (1, MYSEQUENCE.nextVal)"); + @Test public void testInsertWithSequnce() throws Exception { + helpTestVisitor("insert into test.group (e0) values (1)", "INSERT INTO group (e0, e1) VALUES (1, MYSEQUENCE.nextVal)"); //$NON-NLS-1$ //$NON-NLS-2$ } - public void testInsertWithSequnce1() throws Exception { - helpTestVisitor("insert into test.group (e0, e1) values (1, 'x')", "INSERT INTO group (e0, e1) VALUES (1, 'x')"); + @Test public void testInsertWithSequnce1() throws Exception { + helpTestVisitor("insert into test.group (e0, e1) values (1, 'x')", "INSERT INTO group (e0, e1) VALUES (1, 'x')"); //$NON-NLS-1$ //$NON-NLS-2$ } - public void testJoins() throws Exception { + @Test public void testJoins() throws Exception { String input = "select smalla.intkey from bqt1.smalla inner join bqt1.smallb on smalla.stringkey=smallb.stringkey cross join bqt1.mediuma"; //$NON-NLS-1$ String output = "SELECT SmallA.IntKey FROM SmallA INNER JOIN SmallB ON SmallA.StringKey = SmallB.StringKey CROSS JOIN MediumA"; //$NON-NLS-1$ @@ -76,7 +87,7 @@ output, TRANSLATOR); } - public void testJoins2() throws Exception { + @Test public void testJoins2() throws Exception { String input = "select smalla.intkey from bqt1.smalla cross join (bqt1.smallb cross join bqt1.mediuma)"; //$NON-NLS-1$ String output = "SELECT SmallA.IntKey FROM SmallA CROSS JOIN (SmallB CROSS JOIN MediumA)"; //$NON-NLS-1$ @@ -84,4 +95,199 @@ input, output, TRANSLATOR); } + + @Test public void testRewriteConversion1() throws Exception { + String input = "SELECT char(convert(STRINGNUM, integer) + 100) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT chr((to_number(SmallA.StringNum) + 100)) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testRewriteConversion2() throws Exception { + String input = "SELECT convert(STRINGNUM, long) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_number(SmallA.StringNum) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testRewriteConversion3() throws Exception { + String input = "SELECT convert(convert(STRINGNUM, long), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(to_number(SmallA.StringNum)) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testRewriteConversion4() throws Exception { + String input = "SELECT convert(convert(TIMESTAMPVALUE, date), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(trunc(SmallA.TimestampValue), 'YYYY-MM-DD') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion5() throws Exception { + String input = "SELECT convert(convert(TIMESTAMPVALUE, time), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(to_date(('1970-01-01 ' || to_char(SmallA.TimestampValue, 'HH24:MI:SS')), 'YYYY-MM-DD HH24:MI:SS'), 'HH24:MI:SS') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion6() throws Exception { + String input = "SELECT convert(convert(TIMEVALUE, timestamp), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(cast(SmallA.TimeValue AS timestamp), 'YYYY-MM-DD HH24:MI:SS.FF') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion8() throws Exception { + String input = "SELECT nvl(INTNUM, 'otherString') FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT nvl(to_char(SmallA.IntNum), 'otherString') FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteConversion7() throws Exception { + String input = "SELECT convert(convert(STRINGNUM, integer), string) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT to_char(to_number(SmallA.StringNum)) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Ignore("TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is") + @Test public void testRewriteLocate() throws Exception { + // TODO TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is + String input = "SELECT locate(INTNUM, 'chimp', 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT instr('chimp', to_char(SmallA.IntNum), 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteLocate2() throws Exception { + String input = "SELECT locate(STRINGNUM, 'chimp') FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT instr('chimp', SmallA.StringNum) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Ignore("TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is") + @Test public void testRewriteLocate3() throws Exception { + // TODO TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is used as is + String input = "SELECT locate(INTNUM, '234567890', 1) FROM BQT1.SMALLA WHERE INTKEY = 26"; //$NON-NLS-1$ + String output = "SELECT instr('234567890', to_char(SmallA.IntNum), 1) FROM SmallA WHERE SmallA.IntKey = 26"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteLocate4() throws Exception { + String input = "SELECT locate('c', 'chimp', 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT 1 FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Ignore("TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is 1 if a value of < 1 is given") + @Test public void testRewriteLocate5() throws Exception { + // TODO TEIID-754: Fix Oracle translator so fromPosition of LOCATE function is 1 if a value of < 1 is given + String input = "SELECT locate(STRINGNUM, 'chimp', -5) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT instr('chimp', SmallA.StringNum, 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteSubstring1() throws Exception { + String input = "SELECT substring(StringNum, 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT substr(SmallA.StringNum, 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteSubstring2() throws Exception { + String input = "SELECT substring(StringNum, 1, 1) FROM BQT1.SMALLA"; //$NON-NLS-1$ + String output = "SELECT substr(SmallA.StringNum, 1, 1) FROM SmallA"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRewriteUnionWithOrderBy() throws Exception { + String input = "SELECT IntKey FROM BQT1.SMALLA UNION SELECT IntKey FROM BQT1.SMALLB ORDER BY IntKey"; //$NON-NLS-1$ + String output = "SELECT SmallA.IntKey FROM SmallA UNION SELECT SmallB.IntKey FROM SmallB ORDER BY IntKey NULLS FIRST"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit1() throws Exception { + String input = "select intkey from bqt1.smalla limit 10, 0"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT VIEW_FOR_LIMIT.*, ROWNUM ROWNUM_ FROM (SELECT SmallA.IntKey FROM SmallA) VIEW_FOR_LIMIT WHERE ROWNUM <= 10) WHERE ROWNUM_ > 10"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit2() throws Exception { + String input = "select intkey from bqt1.smalla limit 0, 10"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT SmallA.IntKey FROM SmallA) WHERE ROWNUM <= 10"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit3() throws Exception { + String input = "select intkey from bqt1.smalla limit 1, 10"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT VIEW_FOR_LIMIT.*, ROWNUM ROWNUM_ FROM (SELECT SmallA.IntKey FROM SmallA) VIEW_FOR_LIMIT WHERE ROWNUM <= 11) WHERE ROWNUM_ > 1"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit4() throws Exception { + String input = "select intkey from bqt1.mediuma limit 100"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT MediumA.IntKey FROM MediumA) WHERE ROWNUM <= 100"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + @Test public void testRowLimit5() throws Exception { + String input = "select intkey from bqt1.mediuma limit 50, 100"; //$NON-NLS-1$ + String output = "SELECT * FROM (SELECT VIEW_FOR_LIMIT.*, ROWNUM ROWNUM_ FROM (SELECT MediumA.IntKey FROM MediumA) VIEW_FOR_LIMIT WHERE ROWNUM <= 150) WHERE ROWNUM_ > 50"; //$NON-NLS-1$ + + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testConcat2_useLiteral() throws Exception { + String input = "select concat2(stringnum,'_xx') from bqt1.Smalla"; //$NON-NLS-1$ + String output = "SELECT concat(nvl(SmallA.StringNum, ''), '_xx') FROM SmallA"; //$NON-NLS-1$ + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + + @Test public void testConcat2() throws Exception { + String input = "select concat2(stringnum, stringkey) from bqt1.Smalla"; //$NON-NLS-1$ + String output = "SELECT CASE WHEN (SmallA.StringNum IS NULL) AND (SmallA.StringKey IS NULL) THEN NULL ELSE concat(nvl(SmallA.StringNum, ''), nvl(SmallA.StringKey, '')) END FROM SmallA"; //$NON-NLS-1$ + MetadataFactory.helpTestVisitor(MetadataFactory.BQT_VDB, + input, output, + TRANSLATOR); + } + }

16 years, 8 months

1
0
0 / 0

teiid SVN: r1227 - in trunk: connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc and 1 other directories.

by teiid-commits＠lists.jboss.org

Author: shawkins Date: 2009-08-10 14:26:58 -0400 (Mon, 10 Aug 2009) New Revision: 1227 Added: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/TestJDBCProcedureExecution.java Modified: trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/JDBCProcedureExecution.java trunk/engine/src/test/java/com/metamatrix/query/unittest/FakeMetadataFactory.java Log: TEIID-765 fixing indexing error with procedure execution Modified: trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/JDBCProcedureExecution.java =================================================================== --- trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/JDBCProcedureExecution.java 2009-08-10 16:42:31 UTC (rev 1226) +++ trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/JDBCProcedureExecution.java 2009-08-10 18:26:58 UTC (rev 1227) @@ -110,7 +110,7 @@ int paramIndex = 1; for (IParameter parameter : proc.getParameters()) { if (parameter.getDirection() == Direction.RETURN) { - addParameterValue(result, paramIndex, parameter); + addParameterValue(result, paramIndex++, parameter); break; } } @@ -118,9 +118,8 @@ if (parameter.getDirection() == Direction.RETURN || parameter.getDirection() == Direction.RESULT_SET) { continue; } - paramIndex++; if (parameter.getDirection() == Direction.INOUT || parameter.getDirection() == Direction.OUT) { - addParameterValue(result, paramIndex, parameter); + addParameterValue(result, paramIndex++, parameter); } } return result; Added: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/TestJDBCProcedureExecution.java =================================================================== --- trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/TestJDBCProcedureExecution.java (rev 0) +++ trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/TestJDBCProcedureExecution.java 2009-08-10 18:26:58 UTC (rev 1227) @@ -0,0 +1,62 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.connector.jdbc; + +import static org.junit.Assert.*; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.Types; +import java.util.Arrays; +import java.util.Properties; + +import org.junit.Test; +import org.mockito.Mockito; +import org.teiid.connector.api.ConnectorLogger; +import org.teiid.connector.api.ExecutionContext; +import org.teiid.connector.jdbc.translator.Translator; +import org.teiid.connector.language.ICommand; +import org.teiid.dqp.internal.datamgr.metadata.RuntimeMetadataImpl; + +import com.metamatrix.cdk.api.EnvironmentUtility; +import com.metamatrix.query.unittest.FakeMetadataFactory; + +public class TestJDBCProcedureExecution { + + @Test public void testProcedureExecution() throws Exception { + ICommand command = MetadataFactory.helpTranslate(MetadataFactory.BQT_VDB, "exec pm2.spTest8a()"); //$NON-NLS-1$ + Connection connection = Mockito.mock(Connection.class); + CallableStatement cs = Mockito.mock(CallableStatement.class); + Mockito.stub(cs.getUpdateCount()).toReturn(-1); + Mockito.stub(cs.getInt(1)).toReturn(5); + Mockito.stub(connection.prepareCall("{ call spTest8a(?)}")).toReturn(cs); //$NON-NLS-1$ + Translator sqlTranslator = new Translator(); + ExecutionContext context = EnvironmentUtility.createSecurityContext("user"); //$NON-NLS-1$ + RuntimeMetadataImpl runtimeMetadata = new RuntimeMetadataImpl(FakeMetadataFactory.exampleBQTCached()); + JDBCProcedureExecution procedureExecution = new JDBCProcedureExecution(command, connection, sqlTranslator, Mockito.mock(ConnectorLogger.class), new Properties(), runtimeMetadata, context, EnvironmentUtility.createEnvironment(new Properties()) ); + procedureExecution.execute(); + assertEquals(Arrays.asList(5), procedureExecution.getOutputParameterValues()); + Mockito.verify(cs, Mockito.times(1)).registerOutParameter(1, Types.INTEGER); + } + +} Property changes on: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/TestJDBCProcedureExecution.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/engine/src/test/java/com/metamatrix/query/unittest/FakeMetadataFactory.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/unittest/FakeMetadataFactory.java 2009-08-10 16:42:31 UTC (rev 1226) +++ trunk/engine/src/test/java/com/metamatrix/query/unittest/FakeMetadataFactory.java 2009-08-10 18:26:58 UTC (rev 1227) @@ -2178,6 +2178,8 @@ FakeMetadataObject rs2p3 = createParameter("ret", 1, ParameterInfo.RESULT_SET, DataTypeManager.DefaultDataTypes.OBJECT, rs2); //$NON-NLS-1$ FakeMetadataObject spTest8 = createStoredProcedure("pm2.spTest8", pm2, Arrays.asList(new FakeMetadataObject[] { rs2p1, rs2p2, rs2p3}), "spTest8"); //$NON-NLS-1$ //$NON-NLS-2$ + FakeMetadataObject spTest8a = createStoredProcedure("pm2.spTest8a", pm2, Arrays.asList(new FakeMetadataObject[] { rs2p2, rs2p3}), "spTest8a"); //$NON-NLS-1$ //$NON-NLS-2$ + FakeMetadataObject pm4 = createPhysicalModel("pm4"); //$NON-NLS-1$ FakeMetadataObject rs4p1 = createParameter("inkey", 2, ParameterInfo.IN, DataTypeManager.DefaultDataTypes.INTEGER, null); //$NON-NLS-1$ FakeMetadataObject rs4p2 = createParameter("ret", 1, ParameterInfo.RETURN_VALUE, DataTypeManager.DefaultDataTypes.INTEGER, null); //$NON-NLS-1$ @@ -2291,6 +2293,7 @@ store.addObject(pm2); store.addObject(spTest8); + store.addObject(spTest8a); store.addObject(rs2); store.addObject(pm4);

16 years, 8 months

1
0
0 / 0

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

teiid-commits August 2009