teiid SVN: r1765 - in trunk/connectors/connector-jdbc/src: test/java/org/teiid/connector/jdbc/mysql and 1 other directory.
by teiid-commits＠lists.jboss.org
Author: shawkins
Date: 2010-01-21 10:42:22 -0500 (Thu, 21 Jan 2010)
New Revision: 1765
Added:
trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQL5Translator.java
Modified:
trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/mysql/MySQL5Translator.java
trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQLTranslator.java
Log:
TEIID-938 fix for mysql 5 char function
Modified: trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/mysql/MySQL5Translator.java
===================================================================
--- trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/mysql/MySQL5Translator.java 2010-01-21 05:11:37 UTC (rev 1764)
+++ trunk/connectors/connector-jdbc/src/main/java/org/teiid/connector/jdbc/mysql/MySQL5Translator.java 2010-01-21 15:42:22 UTC (rev 1765)
@@ -22,11 +22,31 @@
package org.teiid.connector.jdbc.mysql;
+import java.util.Arrays;
+import java.util.List;
+
import org.teiid.connector.api.ConnectorCapabilities;
+import org.teiid.connector.api.ConnectorEnvironment;
+import org.teiid.connector.api.ConnectorException;
+import org.teiid.connector.api.SourceSystemFunctions;
+import org.teiid.connector.jdbc.translator.FunctionModifier;
+import org.teiid.connector.language.IFunction;
-public class MySQL5Translator extends MySQLTranslator {
+public class MySQL5Translator extends MySQLTranslator {
@Override
+ public void initialize(ConnectorEnvironment env) throws ConnectorException {
+ super.initialize(env);
+ registerFunctionModifier(SourceSystemFunctions.CHAR, new FunctionModifier() {
+
+ @Override
+ public List<?> translate(IFunction function) {
+ return Arrays.asList("char(", function.getParameters().get(0), " USING ASCII)"); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ });
+ }
+
+ @Override
public Class<? extends ConnectorCapabilities> getDefaultCapabilities() {
return MySQL5Capabilities.class;
}
Added: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQL5Translator.java
===================================================================
--- trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQL5Translator.java (rev 0)
+++ trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQL5Translator.java 2010-01-21 15:42:22 UTC (rev 1765)
@@ -0,0 +1,54 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.connector.jdbc.mysql;
+
+import java.util.Properties;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.teiid.connector.api.ConnectorException;
+import org.teiid.connector.jdbc.TranslationHelper;
+
+import com.metamatrix.cdk.api.EnvironmentUtility;
+
+/**
+ */
+public class TestMySQL5Translator {
+
+ private static MySQL5Translator TRANSLATOR;
+
+ @BeforeClass public static void oneTimeSetup() throws ConnectorException {
+ TRANSLATOR = new MySQL5Translator();
+ TRANSLATOR.initialize(EnvironmentUtility.createEnvironment(new Properties(), false));
+ }
+
+ @Test public void testChar() throws Exception {
+ String input = "SELECT intkey, CHR(CONVERT(bigintegervalue, integer)) FROM BQT1.MediumA"; //$NON-NLS-1$
+ String output = "SELECT MediumA.IntKey, char(cast(MediumA.BigIntegerValue AS signed) USING ASCII) FROM MediumA"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(TranslationHelper.BQT_VDB,
+ input,
+ output, TRANSLATOR);
+ }
+
+}
Property changes on: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQL5Translator.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Modified: trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQLTranslator.java
===================================================================
--- trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQLTranslator.java 2010-01-21 05:11:37 UTC (rev 1764)
+++ trunk/connectors/connector-jdbc/src/test/java/org/teiid/connector/jdbc/mysql/TestMySQLTranslator.java 2010-01-21 15:42:22 UTC (rev 1765)
@@ -20,8 +20,8 @@
* 02110-1301 USA.
*/
-package org.teiid.connector.jdbc.mysql;
-
+package org.teiid.connector.jdbc.mysql;
+
import java.util.Properties;
import org.junit.BeforeClass;
@@ -30,102 +30,102 @@
import org.teiid.connector.jdbc.TranslationHelper;
import com.metamatrix.cdk.api.EnvironmentUtility;
-
-/**
- */
-public class TestMySQLTranslator {
-
- private static MySQLTranslator TRANSLATOR;
+
+/**
+ */
+public class TestMySQLTranslator {
+
+ private static MySQLTranslator TRANSLATOR;
- @BeforeClass public static void oneTimeSetup() throws ConnectorException {
- TRANSLATOR = new MySQLTranslator();
+ @BeforeClass public static void oneTimeSetup() throws ConnectorException {
+ TRANSLATOR = new MySQLTranslator();
TRANSLATOR.initialize(EnvironmentUtility.createEnvironment(new Properties(), false));
- }
-
- private String getTestVDB() {
- return TranslationHelper.PARTS_VDB;
- }
-
- private String getTestBQTVDB() {
- return TranslationHelper.BQT_VDB;
- }
-
+ }
+
+ private String getTestVDB() {
+ return TranslationHelper.PARTS_VDB;
+ }
+
+ private String getTestBQTVDB() {
+ return TranslationHelper.BQT_VDB;
+ }
+
@Test public void testConversion1() throws Exception {
- String input = "SELECT char(convert(PART_WEIGHT, integer) + 100) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT char((cast(PARTS.PART_WEIGHT AS signed) + 100)) FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
-
+ String input = "SELECT char(convert(PART_WEIGHT, integer) + 100) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT char((cast(PARTS.PART_WEIGHT AS signed) + 100)) FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
+
@Test public void testConversion2() throws Exception {
- String input = "SELECT convert(PART_WEIGHT, long) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT cast(PARTS.PART_WEIGHT AS signed) FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
-
+ String input = "SELECT convert(PART_WEIGHT, long) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT cast(PARTS.PART_WEIGHT AS signed) FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
+
@Test public void testConversion3() throws Exception {
- String input = "SELECT convert(convert(PART_WEIGHT, long), string) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT cast(cast(PARTS.PART_WEIGHT AS signed) AS char) FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
-
+ String input = "SELECT convert(convert(PART_WEIGHT, long), string) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT cast(cast(PARTS.PART_WEIGHT AS signed) AS char) FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
+
@Test public void testConversion4() throws Exception {
- String input = "SELECT convert(convert(PART_WEIGHT, date), string) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT date_format(DATE(PARTS.PART_WEIGHT), '%Y-%m-%d') FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT convert(convert(PART_WEIGHT, date), string) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT date_format(DATE(PARTS.PART_WEIGHT), '%Y-%m-%d') FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testConversion5() throws Exception {
- String input = "SELECT convert(convert(PART_WEIGHT, time), string) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT date_format(TIME(PARTS.PART_WEIGHT), '%H:%i:%S') FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT convert(convert(PART_WEIGHT, time), string) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT date_format(TIME(PARTS.PART_WEIGHT), '%H:%i:%S') FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testConversion6() throws Exception {
- String input = "SELECT convert(convert(PART_WEIGHT, timestamp), string) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT date_format(TIMESTAMP(PARTS.PART_WEIGHT), '%Y-%m-%d %H:%i:%S.%f') FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT convert(convert(PART_WEIGHT, timestamp), string) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT date_format(TIMESTAMP(PARTS.PART_WEIGHT), '%Y-%m-%d %H:%i:%S.%f') FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testConversion8() throws Exception {
- String input = "SELECT ifnull(PART_WEIGHT, 'otherString') FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT ifnull(PARTS.PART_WEIGHT, 'otherString') FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT ifnull(PART_WEIGHT, 'otherString') FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT ifnull(PARTS.PART_WEIGHT, 'otherString') FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testConversion7() throws Exception {
- String input = "SELECT convert(convert(PART_WEIGHT, integer), string) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT cast(cast(PARTS.PART_WEIGHT AS signed) AS char) FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT convert(convert(PART_WEIGHT, integer), string) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT cast(cast(PARTS.PART_WEIGHT AS signed) AS char) FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testInsert() throws Exception {
- String input = "SELECT insert(PART_WEIGHT, 1, 5, 'chimp') FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT insert(PARTS.PART_WEIGHT, 1, 5, 'chimp') FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
-
+ String input = "SELECT insert(PART_WEIGHT, 1, 5, 'chimp') FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT insert(PARTS.PART_WEIGHT, 1, 5, 'chimp') FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
+
/**
* Test the translator's ability to rewrite the LOCATE() function in a form
* suitable for the data source.
@@ -209,7 +209,7 @@
TranslationHelper.helpTestVisitor(TranslationHelper.BQT_VDB,
input, output,
TRANSLATOR);
- }
+ }
/**
* Test the translator's ability to rewrite the LOCATE() function in a form
@@ -246,48 +246,48 @@
}
@Test public void testSubstring1() throws Exception {
- String input = "SELECT substring(PART_WEIGHT, 1) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT substring(PARTS.PART_WEIGHT, 1) FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT substring(PART_WEIGHT, 1) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT substring(PARTS.PART_WEIGHT, 1) FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testSubstring2() throws Exception {
- String input = "SELECT substring(PART_WEIGHT, 1, 5) FROM PARTS"; //$NON-NLS-1$
- String output = "SELECT substring(PARTS.PART_WEIGHT, 1, 5) FROM PARTS"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
+ String input = "SELECT substring(PART_WEIGHT, 1, 5) FROM PARTS"; //$NON-NLS-1$
+ String output = "SELECT substring(PARTS.PART_WEIGHT, 1, 5) FROM PARTS"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
+ }
@Test public void testUnionWithOrderBy() throws Exception {
- String input = "SELECT PART_ID FROM PARTS UNION SELECT PART_ID FROM PARTS ORDER BY PART_ID"; //$NON-NLS-1$
- String output = "(SELECT PARTS.PART_ID FROM PARTS) UNION (SELECT PARTS.PART_ID FROM PARTS) ORDER BY PART_ID"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestVDB(),
- input,
- output, TRANSLATOR);
- }
-
- @Test public void testRowLimit2() throws Exception {
- String input = "select intkey from bqt1.smalla limit 100"; //$NON-NLS-1$
- String output = "SELECT SmallA.IntKey FROM SmallA LIMIT 100"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestBQTVDB(),
- input,
- output, TRANSLATOR);
+ String input = "SELECT PART_ID FROM PARTS UNION SELECT PART_ID FROM PARTS ORDER BY PART_ID"; //$NON-NLS-1$
+ String output = "(SELECT PARTS.PART_ID FROM PARTS) UNION (SELECT PARTS.PART_ID FROM PARTS) ORDER BY PART_ID"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestVDB(),
+ input,
+ output, TRANSLATOR);
}
-
- @Test public void testRowLimit3() throws Exception {
- String input = "select intkey from bqt1.smalla limit 50, 100"; //$NON-NLS-1$
- String output = "SELECT SmallA.IntKey FROM SmallA LIMIT 50, 100"; //$NON-NLS-1$
-
- TranslationHelper.helpTestVisitor(getTestBQTVDB(),
- input,
- output, TRANSLATOR);
+
+ @Test public void testRowLimit2() throws Exception {
+ String input = "select intkey from bqt1.smalla limit 100"; //$NON-NLS-1$
+ String output = "SELECT SmallA.IntKey FROM SmallA LIMIT 100"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestBQTVDB(),
+ input,
+ output, TRANSLATOR);
}
+ @Test public void testRowLimit3() throws Exception {
+ String input = "select intkey from bqt1.smalla limit 50, 100"; //$NON-NLS-1$
+ String output = "SELECT SmallA.IntKey FROM SmallA LIMIT 50, 100"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(getTestBQTVDB(),
+ input,
+ output, TRANSLATOR);
+ }
+
@Test public void testBitAnd() throws Exception {
String input = "select bitand(intkey, intnum) from bqt1.smalla"; //$NON-NLS-1$
String output = "SELECT cast((SmallA.IntKey & SmallA.IntNum) AS signed) FROM SmallA"; //$NON-NLS-1$
@@ -322,5 +322,15 @@
TranslationHelper.helpTestVisitor(TranslationHelper.BQT_VDB,
input,
output, TRANSLATOR);
- }
-}
+ }
+
+ @Test public void testChar() throws Exception {
+ String input = "SELECT intkey, CHR(CONVERT(bigintegervalue, integer)) FROM BQT1.MediumA"; //$NON-NLS-1$
+ String output = "SELECT MediumA.IntKey, char(cast(MediumA.BigIntegerValue AS signed)) FROM MediumA"; //$NON-NLS-1$
+
+ TranslationHelper.helpTestVisitor(TranslationHelper.BQT_VDB,
+ input,
+ output, TRANSLATOR);
+ }
+
+}
15 years, 11 months
teiid SVN: r1764 - in trunk/engine/src/main/java/com/metamatrix/common/buffer: impl and 1 other directory.
by teiid-commits＠lists.jboss.org
Author: shawkins
Date: 2010-01-21 00:11:37 -0500 (Thu, 21 Jan 2010)
New Revision: 1764
Modified:
trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java
Log:
TEIID-913 adding back the ability to reserve buffers in the buffermanager and increasing the default for max open files.
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2010-01-21 03:59:24 UTC (rev 1763)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2010-01-21 05:11:37 UTC (rev 1764)
@@ -72,8 +72,7 @@
/**
* Optional property - this values specifies how many open file descriptors should be cached
* in the storage directory. Increasing this value in heavy load may improve performance
- * but will use more file descriptors, which are a limited system resource. The default
- * is 32.
+ * but will use more file descriptors, which are a limited system resource.
*/
public static final String MAX_OPEN_FILES = "metamatrix.buffer.maxOpenFiles"; //$NON-NLS-1$
/**
@@ -88,7 +87,8 @@
public static int DEFAULT_CONNECTOR_BATCH_SIZE = 2048;
public static int DEFAULT_PROCESSOR_BATCH_SIZE = 1024;
- public static int DEFAULT_MAX_PROCESSING_BATCHES = 16;
+ public static int DEFAULT_MAX_PROCESSING_BATCHES = 8;
+ public static int DEFAULT_RESERVE_BUFFERS = 64;
/**
* Get the batch size to use during query processing.
@@ -105,8 +105,35 @@
TupleBuffer createTupleBuffer(List elements, String groupName, TupleSourceType tupleSourceType)
throws MetaMatrixComponentException;
+ /**
+ * Return the maximum number of batches that can be temporarily held potentially
+ * across even a blocked exception.
+ * @return
+ */
int getMaxProcessingBatches();
+ /**
+ * Creates a new {@link FileStore}. See {@link FileStore#setCleanupReference(Object)} to
+ * automatically cleanup the underlying resources.
+ * @param name
+ * @return
+ */
FileStore createFileStore(String name);
+ /**
+ * Reserve up to count buffers for use. Wait will cause the process to block until
+ * all of the requested or half of the total buffers are available.
+ * @param count
+ * @param wait
+ * @return
+ * @throws MetaMatrixComponentException
+ */
+ int reserveBuffers(int count, boolean wait) throws MetaMatrixComponentException;
+
+ /**
+ * Releases the buffers reserved by a call to {@link BufferManager#reserveBuffers(int, boolean)}
+ * @param count
+ */
+ void releaseBuffers(int count);
+
}
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2010-01-21 03:59:24 UTC (rev 1763)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2010-01-21 05:11:37 UTC (rev 1764)
@@ -26,6 +26,8 @@
import java.util.ListIterator;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
import javax.xml.transform.Source;
@@ -60,6 +62,10 @@
private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE;
private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE;
private int maxProcessingBatches = BufferManager.DEFAULT_MAX_PROCESSING_BATCHES;
+ private int reserveBatches = BufferManager.DEFAULT_RESERVE_BUFFERS;
+ private int maxReserveBatches = BufferManager.DEFAULT_RESERVE_BUFFERS;
+ private ReentrantLock lock = new ReentrantLock(true);
+ private Condition batchesFreed = lock.newCondition();
private StorageManager diskMgr;
@@ -169,6 +175,40 @@
}
return types;
}
+
+ @Override
+ public void releaseBuffers(int count) {
+ lock.lock();
+ try {
+ this.reserveBatches += count;
+ batchesFreed.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public int reserveBuffers(int count, boolean wait) throws MetaMatrixComponentException {
+ lock.lock();
+ try {
+ while (wait && count > this.reserveBatches && this.reserveBatches < this.maxReserveBatches / 2) {
+ try {
+ batchesFreed.await();
+ } catch (InterruptedException e) {
+ throw new MetaMatrixComponentException(e);
+ }
+ }
+ this.reserveBatches -= count;
+ if (this.reserveBatches >= 0) {
+ return count;
+ }
+ int result = count + this.reserveBatches;
+ this.reserveBatches = 0;
+ return result;
+ } finally {
+ lock.unlock();
+ }
+ }
public void shutdown() {
}
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2010-01-21 03:59:24 UTC (rev 1763)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2010-01-21 05:11:37 UTC (rev 1764)
@@ -48,7 +48,8 @@
*/
public class FileStorageManager implements StorageManager {
- private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
+ private static final int DEFAULT_MAX_OPEN_FILES = 256;
+ private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
private class FileInfo {
private File file;
@@ -155,7 +156,7 @@
}
// Initialization
- private int maxOpenFiles = 32;
+ private int maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB
private String directory;
private File dirFile;
@@ -200,7 +201,7 @@
}
// Set up max number of open file descriptors
- maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManager.MAX_OPEN_FILES, 32);
+ maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManager.MAX_OPEN_FILES, DEFAULT_MAX_OPEN_FILES);
// Set the max file size
maxFileSize = PropertiesUtils.getIntProperty(props, BufferManager.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB
15 years, 11 months
teiid SVN: r1763 - in trunk/engine/src: test/java/com/metamatrix/query/processor/relational and 1 other directory.
by teiid-commits＠lists.jboss.org
Author: shawkins
Date: 2010-01-20 22:59:24 -0500 (Wed, 20 Jan 2010)
New Revision: 1763
Modified:
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java
Log:
TEIID-925 further refining the dup remove strategy for performance.
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2010-01-21 03:49:48 UTC (rev 1762)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2010-01-21 03:59:24 UTC (rev 1763)
@@ -95,7 +95,7 @@
private List<TupleBuffer> activeTupleBuffers = new ArrayList<TupleBuffer>();
private int masterSortIndex;
- private int dupRemoveSublists = 1; //used to control the number of sublists needed for dup remove
+ private int collected;
// Phase constants for readability
private static final int INITIAL_SORT = 1;
@@ -158,7 +158,8 @@
}
private TupleBuffer createTupleBuffer() throws MetaMatrixComponentException {
- return bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR);
+ TupleBuffer tb = bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR);
+ return tb;
}
/**
@@ -173,48 +174,63 @@
workingTuples = new TreeSet<List<?>>(comparator);
}
}
-
- int maxRows = bufferManager.getMaxProcessingBatches() * bufferManager.getProcessorBatchSize();
- while(!doneReading && workingTuples.size() < maxRows) {
- try {
- List<?> tuple = sourceID.nextTuple();
-
- if (tuple == null) {
- doneReading = true;
- break;
- }
-
- workingTuples.add(tuple);
- } catch(BlockedException e) {
- if ((workingTuples.size() < maxRows/2 && mode != Mode.DUP_REMOVE)
- || (workingTuples.size() < (dupRemoveSublists/4)*bufferManager.getProcessorBatchSize() && activeTupleBuffers.size() < dupRemoveSublists)) {
- throw e; //block if no work can be performed
- }
- break;
- }
- }
-
- if(workingTuples.isEmpty()) {
- break;
- }
+ int totalReservedBuffers = 0;
+ try {
+ int maxRows = bufferManager.getMaxProcessingBatches() * bufferManager.getProcessorBatchSize();
+ while(!doneReading) {
+ if (workingTuples.size() == maxRows) {
+ int reserved = bufferManager.reserveBuffers(1, false);
+ if (reserved == 0) {
+ break;
+ }
+ totalReservedBuffers += 1;
+ maxRows += bufferManager.getProcessorBatchSize();
+ }
+ try {
+ List<?> tuple = sourceID.nextTuple();
+
+ if (tuple == null) {
+ doneReading = true;
+ break;
+ }
+ if (workingTuples.add(tuple)) {
+ this.collected++;
+ }
+ } catch(BlockedException e) {
+ if (mode != Mode.DUP_REMOVE
+ || (this.output != null && collected < this.output.getRowCount() * 2)
+ || (this.output == null && this.workingTuples.isEmpty() && this.activeTupleBuffers.isEmpty())) {
+ throw e; //block if no work can be performed
+ }
+ break;
+ }
+ }
- TupleBuffer sublist = createTupleBuffer();
- activeTupleBuffers.add(sublist);
- if (this.mode == Mode.SORT) {
- //perform a stable sort
- Collections.sort((List<List<?>>)workingTuples, comparator);
- }
- for (List<?> list : workingTuples) {
- sublist.addTuple(list);
- }
- workingTuples = null;
- sublist.saveBatch();
+ if(workingTuples.isEmpty()) {
+ break;
+ }
+
+ TupleBuffer sublist = createTupleBuffer();
+ activeTupleBuffers.add(sublist);
+ if (this.mode == Mode.SORT) {
+ //perform a stable sort
+ Collections.sort((List<List<?>>)workingTuples, comparator);
+ }
+ for (List<?> list : workingTuples) {
+ sublist.addTuple(list);
+ }
+ workingTuples = null;
+
+ sublist.saveBatch();
+ } finally {
+ bufferManager.releaseBuffers(totalReservedBuffers);
+ }
}
if (this.activeTupleBuffers.isEmpty()) {
activeTupleBuffers.add(createTupleBuffer());
}
- this.dupRemoveSublists = Math.min(dupRemoveSublists * 2, bufferManager.getMaxProcessingBatches() * 2);
+ this.collected = 0;
this.phase = MERGE;
}
@@ -224,45 +240,54 @@
TupleBuffer merged = createTupleBuffer();
- int maxSortIndex = Math.min(this.bufferManager.getMaxProcessingBatches() * 2, activeTupleBuffers.size());
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_DQP, "Merging", maxSortIndex, "sublists out of", activeTupleBuffers.size()); //$NON-NLS-1$ //$NON-NLS-2$
+ int maxSortIndex = Math.min(activeTupleBuffers.size(), this.bufferManager.getMaxProcessingBatches());
+ int reservedBuffers = 0;
+ if (activeTupleBuffers.size() > maxSortIndex) {
+ reservedBuffers = bufferManager.reserveBuffers(activeTupleBuffers.size() - maxSortIndex, true);
}
- // initialize the sublists with the min value
- for(int i = 0; i<maxSortIndex; i++) {
- TupleBuffer activeID = activeTupleBuffers.get(i);
- SortedSublist sortedSublist = new SortedSublist();
- sortedSublist.its = activeID.createIndexedTupleSource();
- sortedSublist.index = i;
- if (activeID == output) {
- sortedSublist.limit = output.getRowCount();
- }
- incrementWorkingTuple(sublists, sortedSublist);
+ maxSortIndex += reservedBuffers;
+ try {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_DQP, "Merging", maxSortIndex, "sublists out of", activeTupleBuffers.size()); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ // initialize the sublists with the min value
+ for(int i = 0; i<maxSortIndex; i++) {
+ TupleBuffer activeID = activeTupleBuffers.get(i);
+ SortedSublist sortedSublist = new SortedSublist();
+ sortedSublist.its = activeID.createIndexedTupleSource();
+ sortedSublist.index = i;
+ if (activeID == output) {
+ sortedSublist.limit = output.getRowCount();
+ }
+ incrementWorkingTuple(sublists, sortedSublist);
+ }
+
+ // iteratively process the lowest tuple
+ while (sublists.size() > 0) {
+ SortedSublist sortedSublist = sublists.remove(sublists.size() - 1);
+ merged.addTuple(sortedSublist.tuple);
+ if (this.output != null && sortedSublist.index > masterSortIndex) {
+ this.output.addTuple(sortedSublist.tuple); //a new distinct row
+ }
+ incrementWorkingTuple(sublists, sortedSublist);
+ }
+
+ // Remove merged sublists
+ for(int i=0; i<maxSortIndex; i++) {
+ TupleBuffer id = activeTupleBuffers.remove(0);
+ if (id != this.output) {
+ id.remove();
+ }
+ }
+ merged.saveBatch();
+ this.activeTupleBuffers.add(merged);
+ masterSortIndex = masterSortIndex - maxSortIndex + 1;
+ if (masterSortIndex < 0) {
+ masterSortIndex = this.activeTupleBuffers.size() - 1;
+ }
+ } finally {
+ this.bufferManager.releaseBuffers(reservedBuffers);
}
-
- // iteratively process the lowest tuple
- while (sublists.size() > 0) {
- SortedSublist sortedSublist = sublists.remove(sublists.size() - 1);
- merged.addTuple(sortedSublist.tuple);
- if (this.output != null && sortedSublist.index > masterSortIndex) {
- this.output.addTuple(sortedSublist.tuple); //a new distinct row
- }
- incrementWorkingTuple(sublists, sortedSublist);
- }
-
- // Remove merged sublists
- for(int i=0; i<maxSortIndex; i++) {
- TupleBuffer id = activeTupleBuffers.remove(0);
- if (id != this.output) {
- id.remove();
- }
- }
- merged.saveBatch();
- this.activeTupleBuffers.add(merged);
- masterSortIndex = masterSortIndex - maxSortIndex + 1;
- if (masterSortIndex < 0) {
- masterSortIndex = this.activeTupleBuffers.size() - 1;
- }
}
// Close sorted source (all others have been removed)
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2010-01-21 03:49:48 UTC (rev 1762)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2010-01-21 03:59:24 UTC (rev 1763)
@@ -295,6 +295,7 @@
}
tsid.addTuple(Arrays.asList(2));
+ tsid.addTuple(Arrays.asList(3));
su.sort();
assertEquals(Arrays.asList(2), ts.nextTuple());
}
15 years, 11 months
teiid SVN: r1762 - in trunk/client/src: main/java/org/teiid/netty/handler/codec/serialization and 1 other directories.
by teiid-commits＠lists.jboss.org
Author: shawkins
Date: 2010-01-20 22:49:48 -0500 (Wed, 20 Jan 2010)
New Revision: 1762
Modified:
trunk/client/src/main/java/com/metamatrix/common/comm/platform/socket/client/SocketServerInstanceImpl.java
trunk/client/src/main/java/org/teiid/netty/handler/codec/serialization/ObjectDecoderInputStream.java
trunk/client/src/test/java/org/teiid/netty/handler/codec/serialization/TestObjectDecoderInputStream.java
Log:
TEIID-916 fix for stream corruption during a timeout.
Modified: trunk/client/src/main/java/com/metamatrix/common/comm/platform/socket/client/SocketServerInstanceImpl.java
===================================================================
--- trunk/client/src/main/java/com/metamatrix/common/comm/platform/socket/client/SocketServerInstanceImpl.java 2010-01-20 20:25:36 UTC (rev 1761)
+++ trunk/client/src/main/java/com/metamatrix/common/comm/platform/socket/client/SocketServerInstanceImpl.java 2010-01-21 03:49:48 UTC (rev 1762)
@@ -41,7 +41,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -84,7 +83,7 @@
private Map<Serializable, ResultsReceiver<Object>> asynchronousListeners = new ConcurrentHashMap<Serializable, ResultsReceiver<Object>>();
- private ReentrantLock readLock = new ReentrantLock();
+ private boolean hasReader;
public SocketServerInstanceImpl() {
@@ -318,18 +317,29 @@
TimeoutException {
long timeoutMillis = (int)Math.min(unit.toMillis(timeout), Integer.MAX_VALUE);
long start = System.currentTimeMillis();
- boolean reading = false;
while (!isDone()) {
- try {
- if ((reading = readLock.tryLock(timeoutMillis, TimeUnit.MILLISECONDS)) == true && !isDone()) {
- receivedMessage(socketChannel.read());
+ boolean reading = false;
+ synchronized (SocketServerInstanceImpl.this) {
+ if (!hasReader) {
+ hasReader = true;
+ reading = true;
+ } else if (!isDone()) {
+ SocketServerInstanceImpl.this.wait(Math.max(1, timeoutMillis));
}
- } catch (SocketTimeoutException e) {
- } catch (Exception e) {
- exceptionOccurred(e);
- } finally {
- if (reading) {
- readLock.unlock();
+ }
+ if (reading) {
+ try {
+ if (!isDone()) {
+ receivedMessage(socketChannel.read());
+ }
+ } catch (SocketTimeoutException e) {
+ } catch (Exception e) {
+ exceptionOccurred(e);
+ } finally {
+ synchronized (SocketServerInstanceImpl.this) {
+ hasReader = false;
+ SocketServerInstanceImpl.this.notifyAll();
+ }
}
}
if (!isDone()) {
Modified: trunk/client/src/main/java/org/teiid/netty/handler/codec/serialization/ObjectDecoderInputStream.java
===================================================================
--- trunk/client/src/main/java/org/teiid/netty/handler/codec/serialization/ObjectDecoderInputStream.java 2010-01-20 20:25:36 UTC (rev 1761)
+++ trunk/client/src/main/java/org/teiid/netty/handler/codec/serialization/ObjectDecoderInputStream.java 2010-01-21 03:49:48 UTC (rev 1762)
@@ -65,7 +65,7 @@
buffer = new byte[4];
}
fillBuffer();
- int dataLen = ((buffer[0] & 0xff << 24) + (buffer[1] & 0xff << 16) + (buffer[2] & 0xff << 8) + (buffer[3] & 0xff << 0));
+ int dataLen = getIntFromBytes(buffer);
if (dataLen <= 0) {
throw new StreamCorruptedException("invalid data length: " + dataLen); //$NON-NLS-1$
}
@@ -87,6 +87,10 @@
return new CompactObjectInputStream(bais, classLoader).readObject();
}
+ static int getIntFromBytes(byte[] buffer) {
+ return ((buffer[0] & 0xff) << 24) + ((buffer[1] & 0xff) << 16) + ((buffer[2] & 0xff) << 8) + (buffer[3] & 0xff);
+ }
+
private void fillBuffer() throws IOException, EOFException {
while (count < buffer.length) {
int read = in.read(buffer, count, buffer.length - count);
Modified: trunk/client/src/test/java/org/teiid/netty/handler/codec/serialization/TestObjectDecoderInputStream.java
===================================================================
--- trunk/client/src/test/java/org/teiid/netty/handler/codec/serialization/TestObjectDecoderInputStream.java 2010-01-20 20:25:36 UTC (rev 1761)
+++ trunk/client/src/test/java/org/teiid/netty/handler/codec/serialization/TestObjectDecoderInputStream.java 2010-01-21 03:49:48 UTC (rev 1762)
@@ -67,4 +67,13 @@
assertEquals(obj, result);
}
+ @Test public void testLargeIntConversion() throws Exception {
+ int testValue = 204503404;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ dos.writeInt(testValue);
+ dos.close();
+ assertEquals(testValue, ObjectDecoderInputStream.getIntFromBytes(baos.toByteArray()));
+ }
+
}
15 years, 11 months
teiid SVN: r1761 - trunk/console.
by teiid-commits＠lists.jboss.org
Author: tejones
Date: 2010-01-20 15:25:36 -0500 (Wed, 20 Jan 2010)
New Revision: 1761
Modified:
trunk/console/pom.xml
Log:
Updated Teiid version
Modified: trunk/console/pom.xml
===================================================================
--- trunk/console/pom.xml 2010-01-20 20:22:54 UTC (rev 1760)
+++ trunk/console/pom.xml 2010-01-20 20:25:36 UTC (rev 1761)
@@ -5,7 +5,7 @@
<parent>
<artifactId>teiid</artifactId>
<groupId>org.jboss.teiid</groupId>
- <version>6.3.0-SNAPSHOT</version>
+ <version>7.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
15 years, 11 months
teiid SVN: r1760 - in trunk/engine/src/main/java: org/teiid/dqp/internal/process and 1 other directory.
by teiid-commits＠lists.jboss.org
Author: shawkins
Date: 2010-01-20 15:22:54 -0500 (Wed, 20 Jan 2010)
New Revision: 1760
Modified:
trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
Log:
TEIID-936 fixing the timeslicing mechanism to prevent query hangs
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2010-01-20 18:29:36 UTC (rev 1759)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2010-01-20 20:22:54 UTC (rev 1760)
@@ -32,6 +32,7 @@
import com.metamatrix.common.buffer.TupleBatch;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
import com.metamatrix.common.log.LogManager;
+import com.metamatrix.core.MetaMatrixRuntimeException;
import com.metamatrix.core.log.MessageLevel;
import com.metamatrix.core.util.Assertion;
import com.metamatrix.dqp.util.LogConstants;
@@ -41,6 +42,12 @@
public class QueryProcessor implements BatchProducer {
+ public static class ExpiredTimeSliceException extends MetaMatrixRuntimeException {
+
+ }
+
+ private static ExpiredTimeSliceException EXPIRED_TIME_SLICE = new ExpiredTimeSliceException();
+
public interface ProcessorFactory {
QueryProcessor createQueryProcessor(String query, String recursionGroup, CommandContext commandContext) throws MetaMatrixProcessingException, MetaMatrixComponentException;
}
@@ -92,16 +99,19 @@
while (true) {
try {
return nextBatchDirect();
+ } catch (ExpiredTimeSliceException e) {
+ if (!nonBlocking) {
+ throw e;
+ }
} catch (BlockedException e) {
- if (nonBlocking) {
- try {
- Thread.sleep(DEFAULT_WAIT);
- } catch (InterruptedException err) {
- throw new MetaMatrixComponentException(err);
- }
- continue;
+ if (!nonBlocking) {
+ throw e;
}
- throw e;
+ try {
+ Thread.sleep(DEFAULT_WAIT);
+ } catch (InterruptedException err) {
+ throw new MetaMatrixComponentException(err);
+ }
}
}
}
@@ -162,7 +172,7 @@
closeProcessing();
}
if (result == null) {
- throw BlockedException.INSTANCE; //expired timeslice
+ throw EXPIRED_TIME_SLICE;
}
return result;
}
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2010-01-20 18:29:36 UTC (rev 1759)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2010-01-20 20:22:54 UTC (rev 1760)
@@ -187,6 +187,9 @@
}
} catch (BlockedException e) {
LogManager.logDetail(LogConstants.CTX_DQP, "############# PW EXITING on", requestID, "- processor blocked ###########"); //$NON-NLS-1$ //$NON-NLS-2$
+ } catch (QueryProcessor.ExpiredTimeSliceException e) {
+ LogManager.logDetail(LogConstants.CTX_DQP, "############# PW reenqueueing ", requestID, "- time slice expired ###########"); //$NON-NLS-1$ //$NON-NLS-2$
+ this.moreWork();
} catch (Throwable e) {
LogManager.logDetail(LogConstants.CTX_DQP, e, "############# PW EXITING on", requestID, "- error occurred ###########"); //$NON-NLS-1$ //$NON-NLS-2$
@@ -490,13 +493,10 @@
resultsReceiver.receiveResults(response);
}
- private static List getParameterInfo(StoredProcedure procedure) {
- List params = procedure.getParameters();
- List paramInfos = new ArrayList(params.size());
+ private static List<ParameterInfo> getParameterInfo(StoredProcedure procedure) {
+ List<ParameterInfo> paramInfos = new ArrayList<ParameterInfo>();
- Iterator iter = params.iterator();
- while(iter.hasNext()) {
- SPParameter param = (SPParameter) iter.next();
+ for (SPParameter param : procedure.getParameters()) {
ParameterInfo info = new ParameterInfo(param.getParameterType(), param.getResultSetColumns().size());
paramInfos.add(info);
}
15 years, 11 months
