1 p.m.
Author: shawkins
Date: 2010-01-11 14:00:14 -0500 (Mon, 11 Jan 2010)
New Revision: 1729
Modified:
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java
Log:
TEIID-926 TEIID-925 simplifying projection to simple indexing and correcting the dup
removal logic in sort utility/sort node. also modifying the dup removal algorithm to
force progressively more sublists to be present before performing a merge.
Modified:
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java
===================================================================
---
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java 2010-01-10
05:42:02 UTC (rev 1728)
+++
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java 2010-01-11
19:00:14 UTC (rev 1729)
@@ -39,7 +39,6 @@
import com.metamatrix.core.log.MessageLevel;
import com.metamatrix.core.util.Assertion;
import com.metamatrix.dqp.util.LogConstants;
-import com.metamatrix.query.execution.QueryExecPlugin;
import com.metamatrix.query.processor.Describable;
import com.metamatrix.query.processor.DescribableUtil;
import com.metamatrix.query.processor.ProcessorDataManager;
@@ -51,7 +50,6 @@
import com.metamatrix.query.sql.symbol.SingleElementSymbol;
import com.metamatrix.query.sql.visitor.ValueIteratorProviderCollectorVisitor;
import com.metamatrix.query.util.CommandContext;
-import com.metamatrix.query.util.ErrorMessageKeys;
public abstract class RelationalNode implements Cloneable, Describable, BatchProducer {
Modified:
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java
===================================================================
---
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2010-01-10
05:42:02 UTC (rev 1728)
+++
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2010-01-11
19:00:14 UTC (rev 1729)
@@ -64,7 +64,6 @@
private class SortedSublist implements Comparable<SortedSublist> {
List<?> tuple;
int index;
- boolean duplicate;
IndexedTupleSource its;
int limit = Integer.MAX_VALUE;
@@ -75,7 +74,7 @@
@Override
public String toString() {
- return index + " " + tuple + " " + duplicate; //$NON-NLS-1$
//$NON-NLS-2$
+ return index + " " + tuple; //$NON-NLS-1$
}
}
@@ -202,7 +201,7 @@
if (this.activeTupleBuffers.isEmpty()) {
activeTupleBuffers.add(createTupleBuffer());
}
- this.initialSortPass++;
+ this.initialSortPass = Math.min(initialSortPass + 1,
bufferManager.getMaxProcessingBatches() * 2);
this.phase = MERGE;
}
@@ -244,11 +243,9 @@
// iteratively process the lowest tuple
while (sublists.size() > 0) {
SortedSublist sortedSublist = sublists.remove(0);
- if (this.mode == Mode.SORT || !sortedSublist.duplicate) {
- merged.addTuple(sortedSublist.tuple);
- if (this.output != null && sortedSublist.index >
masterSortIndex) {
- this.output.addTuple(sortedSublist.tuple); //a new distinct row
- }
+ merged.addTuple(sortedSublist.tuple);
+ if (this.output != null && sortedSublist.index >
masterSortIndex) {
+ this.output.addTuple(sortedSublist.tuple); //a new distinct row
}
incrementWorkingTuple(sublists, sortedSublist);
}
@@ -285,52 +282,35 @@
}
private void incrementWorkingTuple(ArrayList<SortedSublist> subLists,
SortedSublist sortedSublist) throws MetaMatrixComponentException,
MetaMatrixProcessingException {
- sortedSublist.tuple = null;
- if (sortedSublist.limit < sortedSublist.its.getCurrentIndex()) {
- return; //special case for still reading the output tuplebuffer
- }
- try {
- sortedSublist.tuple = sortedSublist.its.nextTuple();
- } catch (BlockedException e) {
- //intermediate sources aren't closed
- }
- if (sortedSublist.tuple == null) {
- return; // done with this sublist
- }
- sortedSublist.duplicate = false;
- int index = Collections.binarySearch(subLists, sortedSublist);
- if (index >= 0) {
- /* In dup removal mode we need to ensure that a sublist other than the master
- * gets marked as duplicate
+ while (true) {
+ sortedSublist.tuple = null;
+ if (sortedSublist.limit < sortedSublist.its.getCurrentIndex()) {
+ return; //special case for still reading the output tuplebuffer
+ }
+ try {
+ sortedSublist.tuple = sortedSublist.its.nextTuple();
+ } catch (BlockedException e) {
+ //intermediate sources aren't closed
+ }
+ if (sortedSublist.tuple == null) {
+ return; // done with this sublist
+ }
+ int index = Collections.binarySearch(subLists, sortedSublist);
+ if (index < 0) {
+ subLists.add(-index - 1, sortedSublist);
+ return;
+ }
+ if (mode == Mode.SORT) {
+ subLists.add(index, sortedSublist);
+ return;
+ }
+ /* In dup removal mode we need to ensure that a sublist other than the master is
incremented
*/
if (mode == Mode.DUP_REMOVE && this.output != null &&
sortedSublist.index == masterSortIndex) {
- if (!subLists.get(index).duplicate) {
- subLists.get(index).duplicate = true;
- } else {
- //there's an evil twin somewhere, search before and after the index found
- for (int i = 1; i < subLists.size(); i++) {
- int actualIndex = i;
- if (i <= index) {
- actualIndex = index - i;
- }
- SortedSublist sublist = subLists.get(actualIndex);
- if (sublist.compareTo(sortedSublist) != 0) {
- Assertion.assertTrue(actualIndex < index);
- i = index;
- continue;
- }
- if (!sublist.duplicate) {
- sublist.duplicate = true;
- break;
- }
- }
- }
- } else {
- sortedSublist.duplicate = true;
+ SortedSublist dup = subLists.get(index);
+ subLists.set(index, sortedSublist);
+ sortedSublist = dup;
}
- subLists.add(index, sortedSublist);
- } else {
- subLists.add(-index - 1, sortedSublist);
}
}
Modified:
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java
===================================================================
---
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2010-01-10
05:42:02 UTC (rev 1728)
+++
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2010-01-11
19:00:14 UTC (rev 1729)
@@ -152,9 +152,7 @@
Arrays.sort(expected, comparator);
for (Mode mode : Mode.values()) {
- if (mode == Mode.DUP_REMOVE) {
- helpTestSort(elements, data, sortElements, sortTypes,
mode==Mode.SORT?expected:expectedDistinct, mode);
- }
+ helpTestSort(elements, data, sortElements, sortTypes,
mode==Mode.SORT?expected:expectedDistinct, mode);
}
}