Author: rhauch Date: 2009-11-24 12:44:29 -0500 (Tue, 24 Nov 2009) New Revision: 1339 Added: trunk/dna-search/src/main/java/org/jboss/dna/search/query/IdsQuery.java Removed: trunk/dna-search/src/main/java/org/jboss/dna/search/query/UuidsQuery.java Modified: trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/QueryResults.java trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryProcessor.java trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryResults.java trunk/dna-search/src/main/java/org/jboss/dna/search/DualIndexSearchProvider.java trunk/dna-search/src/main/java/org/jboss/dna/search/IndexRules.java trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneQueryComponent.java trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneSession.java trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareQuery.java trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareStringQuery.java trunk/dna-search/src/test/java/org/jboss/dna/search/SearchEngineTest.java Log: DNA-467 Integrated the Lucene implementation of the SearchEngine component, added a number of unit tests. Modified: trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/QueryResults.java =================================================================== --- trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/QueryResults.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/QueryResults.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -80,6 +80,13 @@ public List<Object[]> getTuples(); /** + * Get the number of rows in the results. + * + * @return the number of rows; never negative + */ + public int getRowCount(); + + /** * Get the problems encountered during execution. * * @return the problems; never null but possibly empty Modified: trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryProcessor.java =================================================================== --- trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryProcessor.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryProcessor.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -162,7 +162,7 @@ switch (node.getType()) { case ACCESS: // If the ACCESS node will not have results ... - if (node.getProperty(Property.ACCESS_NO_RESULTS, Boolean.class)) { + if (node.hasProperty(Property.ACCESS_NO_RESULTS)) { component = new NoResultsComponent(context, columns); } else { // Create the component to handle the ACCESS node ... Modified: trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryResults.java =================================================================== --- trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryResults.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/process/QueryResults.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -138,6 +138,15 @@ /** * {@inheritDoc} * + * @see org.jboss.dna.graph.query.QueryResults#getRowCount() + */ + public int getRowCount() { + return tuples.size(); + } + + /** + * {@inheritDoc} + * * @see org.jboss.dna.graph.query.QueryResults#getProblems() */ public Problems getProblems() { @@ -251,6 +260,7 @@ for (Object[] tuple : getTuples()) { for (int i = 0, j = 1; i != tupleLength; ++i, ++j) { String valueStr = stringOf(tuple[i], stringFactory); + if (valueStr == null) continue; columnWidths[j] = Math.max(Math.min(maxWidth, valueStr.length()), columnWidths[j]); } } Modified: trunk/dna-search/src/main/java/org/jboss/dna/search/DualIndexSearchProvider.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/DualIndexSearchProvider.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/DualIndexSearchProvider.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -28,6 +28,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -60,15 +61,14 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.regex.JavaUtilRegexCapabilities; -import org.apache.lucene.search.regex.RegexQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Version; import org.jboss.dna.common.text.NoOpEncoder; +import org.jboss.dna.common.text.SecureHashTextEncoder; import org.jboss.dna.common.text.TextEncoder; import org.jboss.dna.common.util.Logger; +import org.jboss.dna.common.util.SecureHash.Algorithm; import org.jboss.dna.graph.DnaLexicon; import org.jboss.dna.graph.ExecutionContext; import org.jboss.dna.graph.JcrLexicon; @@ -77,8 +77,8 @@ import org.jboss.dna.graph.property.Binary; import org.jboss.dna.graph.property.DateTime; import org.jboss.dna.graph.property.Name; +import org.jboss.dna.graph.property.NamespaceRegistry; import org.jboss.dna.graph.property.Path; -import org.jboss.dna.graph.property.PathFactory; import org.jboss.dna.graph.property.Property; import org.jboss.dna.graph.property.PropertyType; import org.jboss.dna.graph.property.ValueFactories; @@ -107,13 +107,14 @@ import org.jboss.dna.graph.search.SearchException; import org.jboss.dna.graph.search.SearchProvider; import org.jboss.dna.search.IndexRules.Rule; +import org.jboss.dna.search.LuceneSession.TupleCollector; import org.jboss.dna.search.query.CompareLengthQuery; import org.jboss.dna.search.query.CompareNameQuery; import org.jboss.dna.search.query.ComparePathQuery; import org.jboss.dna.search.query.CompareStringQuery; +import org.jboss.dna.search.query.IdsQuery; import org.jboss.dna.search.query.MatchNoneQuery; import org.jboss.dna.search.query.NotQuery; -import org.jboss.dna.search.query.UuidsQuery; /** * A simple {@link SearchProvider} implementation that relies upon two separate indexes: one for the node content and a second one @@ -134,8 +135,8 @@ static { IndexRules.Builder builder = IndexRules.createBuilder(); // Configure the default behavior ... - builder.defaultTo(IndexRules.INDEX | IndexRules.ANALYZE | IndexRules.FULL_TEXT); - // Configure the UUID properties to be just indexed (not stored, not analyzed, not included in full-text) ... + builder.defaultTo(IndexRules.INDEX | IndexRules.FULL_TEXT | IndexRules.STORE); + // Configure the UUID properties to be just indexed and stored (not analyzed, not included in full-text) ... builder.store(JcrLexicon.UUID, DnaLexicon.UUID); // Configure the properties that we'll treat as dates ... builder.treatAsDates(JcrLexicon.CREATED, JcrLexicon.LAST_MODIFIED); @@ -150,24 +151,40 @@ protected static final double MAX_DOUBLE = Double.MAX_VALUE; protected static final int MIN_DEPTH = 0; protected static final int MAX_DEPTH = 100; + protected static final int MIN_SNS_INDEX = 1; + protected static final int MAX_SNS_INDEX = 1000; // assume there won't be more than 1000 same-name-siblings protected static final String PATHS_INDEX_NAME = "paths"; protected static final String CONTENT_INDEX_NAME = "content"; - protected static final String UUID_FIELD = "uuid"; - protected static final String FULL_TEXT_SUFFIX = "/fs"; // the slash character is not allowed in a property name unescaped + /** + * Given the name of a property field of the form "&lt;namespace>:&lt;local>" (where &lt;namespace> can be zero-length), this + * provider also stores the value(s) for free-text searching in a field named ":ft:&lt;namespace>:&lt;local>". Thus, even if + * the namespace is zero-length, the free-text search field will be named ":ft::&lt;local>" and will not clash with any other + * property name. + */ + protected static final String FULL_TEXT_PREFIX = ":ft:"; + /** + * This index stores only these fields, so we can use the most obvious names and not worry about clashes. + */ static class PathIndex { - public static final String PATH = "path"; - public static final String LOCAL_NAME = "name"; + public static final String PATH = "pth"; + public static final String NODE_NAME = "nam"; + public static final String LOCAL_NAME = "loc"; public static final String SNS_INDEX = "sns"; - public static final String UUID = UUID_FIELD; - public static final String DEPTH = "depth"; + public static final String LOCATION_ID_PROPERTIES = "idp"; + public static final String ID = ContentIndex.ID; + public static final String DEPTH = "dep"; } + /** + * This index stores these two fields <i>plus</i> all properties. Therefore, we have to worry about name clashes, which is why + * these field names are prefixed with '::', which is something that does appear in property names as they are serialized. + */ static class ContentIndex { - public static final String UUID = UUID_FIELD; - public static final String FULL_TEXT = "fts"; + public static final String ID = "::id"; + public static final String FULL_TEXT = "::fts"; } /** @@ -190,12 +207,13 @@ private static final long serialVersionUID = 1L; public FieldSelectorResult accept( String fieldName ) { - return PathIndex.UUID.equals(fieldName) ? FieldSelectorResult.LOAD_AND_BREAK : FieldSelectorResult.NO_LOAD; + return PathIndex.ID.equals(fieldName) ? FieldSelectorResult.LOAD_AND_BREAK : FieldSelectorResult.NO_LOAD; } }; private final IndexRules rules; private final LuceneConfiguration directoryConfiguration; + private final TextEncoder namespaceEncoder; public DualIndexSearchProvider( LuceneConfiguration directoryConfiguration, IndexRules rules ) { @@ -203,6 +221,7 @@ assert rules != null; this.rules = rules; this.directoryConfiguration = directoryConfiguration; + this.namespaceEncoder = new SecureHashTextEncoder(Algorithm.SHA_1, 10); } public DualIndexSearchProvider( LuceneConfiguration directoryConfiguration ) { @@ -253,7 +272,9 @@ assert contentIndexDirectory != null; Analyzer analyzer = createAnalyzer(); assert analyzer != null; - return new DualIndexSession(context, sourceName, workspaceName, rules, pathIndexDirectory, contentIndexDirectory, + NamespaceRegistry encodingRegistry = new EncodingNamespaceRegistry(context.getNamespaceRegistry(), namespaceEncoder); + ExecutionContext encodingContext = context.with(encodingRegistry); + return new DualIndexSession(encodingContext, sourceName, workspaceName, rules, pathIndexDirectory, contentIndexDirectory, analyzer, overwrite, readOnly); } @@ -304,9 +325,82 @@ */ @Override protected String fullTextFieldName( String propertyName ) { - return propertyName + FULL_TEXT_SUFFIX; + return FULL_TEXT_PREFIX + propertyName; } + protected void addIdProperties( Location location, + Document doc ) { + if (!location.hasIdProperties()) return; + for (Property idProp : location.getIdProperties()) { + String fieldValue = serializeProperty(idProp); + doc.add(new Field(PathIndex.LOCATION_ID_PROPERTIES, fieldValue, Field.Store.YES, Field.Index.NOT_ANALYZED)); + } + } + + protected Location readLocation( Document doc ) { + // Read the path ... + String pathString = doc.get(PathIndex.PATH); + Path path = pathFactory.create(pathString); + // Look for the Location's ID properties ... + String[] idProps = doc.getValues(PathIndex.LOCATION_ID_PROPERTIES); + if (idProps.length == 0) { + return Location.create(path); + } + if (idProps.length == 1) { + Property idProp = deserializeProperty(idProps[0]); + if (idProp == null) return Location.create(path); + if (idProp.isSingle() && (idProp.getName().equals(JcrLexicon.UUID) || idProp.getName().equals(DnaLexicon.UUID))) { + return Location.create(path, (UUID)idProp.getFirstValue()); // know that deserialize returns UUID value + } + return Location.create(path, idProp); + } + List<Property> properties = new LinkedList<Property>(); + for (String idProp : doc.getValues(PathIndex.LOCATION_ID_PROPERTIES)) { + Property prop = deserializeProperty(idProp); + if (prop != null) properties.add(prop); + } + return properties.isEmpty() ? Location.create(path) : Location.create(path, properties); + + } + + protected final String serializeProperty( Property property ) { + StringBuilder sb = new StringBuilder(); + sb.append(stringFactory.create(property.getName())); + sb.append('='); + Iterator<?> iter = property.getValues(); + if (iter.hasNext()) { + sb.append(stringFactory.create(iter.next())); + } + while (iter.hasNext()) { + sb.append('\n'); + sb.append(stringFactory.create(iter.next())); + } + return sb.toString(); + } + + protected final Property deserializeProperty( String propertyString ) { + int index = propertyString.indexOf('='); + assert index > -1; + if (index == propertyString.length() - 1) return null; + Name propName = nameFactory.create(propertyString.substring(0, index)); + String valueString = propertyString.substring(index + 1); + // Break into multiple values if multiple lines ... + String[] values = valueString.split("\\n"); + if (values.length == 0) return null; + if (values.length == 1) { + Object value = values[0]; + if (DnaLexicon.UUID.equals(propName) || JcrLexicon.UUID.equals(propName)) { + value = uuidFactory.create(value); + } + return context.getPropertyFactory().create(propName, value); + } + List<String> propValues = new LinkedList<String>(); + for (String value : values) { + propValues.add(value); + } + return context.getPropertyFactory().create(propName, propValues); + } + protected IndexReader getPathsReader() throws IOException { if (pathsReader == null) { pathsReader = IndexReader.open(pathsIndexDirectory, readOnly); @@ -379,9 +473,10 @@ UUID uuid = location.getUuid(); if (uuid == null) uuid = UUID.randomUUID(); Path path = location.getPath(); - String uuidStr = stringFactory.create(uuid); + String idStr = stringFactory.create(uuid); String pathStr = pathAsString(path, stringFactory); String nameStr = path.isRoot() ? "" : stringFactory.create(path.getLastSegment().getName()); + String localNameStr = path.isRoot() ? "" : path.getLastSegment().getName().getLocalName(); int sns = path.isRoot() ? 1 : path.getLastSegment().getIndex(); Logger logger = Logger.getLogger(getClass()); @@ -395,15 +490,17 @@ // be changed without changing any other content fields ... Document doc = new Document(); doc.add(new Field(PathIndex.PATH, pathStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); - doc.add(new Field(PathIndex.LOCAL_NAME, nameStr, Field.Store.YES, Field.Index.ANALYZED)); - doc.add(new NumericField(PathIndex.LOCAL_NAME, Field.Store.YES, true).setIntValue(sns)); - doc.add(new Field(PathIndex.UUID, uuidStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new Field(PathIndex.NODE_NAME, nameStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new Field(PathIndex.LOCAL_NAME, localNameStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new NumericField(PathIndex.SNS_INDEX, Field.Store.YES, true).setIntValue(sns)); + doc.add(new Field(PathIndex.ID, idStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new NumericField(PathIndex.DEPTH, Field.Store.YES, true).setIntValue(path.size())); + addIdProperties(location, doc); getPathsWriter().addDocument(doc); // Create the document for the content (properties) ... doc = new Document(); - doc.add(new Field(ContentIndex.UUID, uuidStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new Field(ContentIndex.ID, idStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); String stringValue = null; StringBuilder fullTextSearchValue = null; for (Property property : node.getProperties()) { @@ -506,12 +603,12 @@ assert !readOnly; try { // Create a query to find all the nodes at or below the specified path ... - Set<UUID> uuids = getUuidsForDescendantsOf(path, true); - Query uuidQuery = findAllNodesWithUuids(uuids); + Set<String> ids = getIdsForDescendantsOf(path, true); + Query uuidQuery = findAllNodesWithIds(ids); // Now delete the documents from each index using this query, which we can reuse ... getPathsWriter().deleteDocuments(uuidQuery); getContentWriter().deleteDocuments(uuidQuery); - return uuids.size(); + return ids.size(); } catch (FileNotFoundException e) { // There are no index files yet, so nothing to delete ... return 0; @@ -551,24 +648,19 @@ int numberOfResults = scoreDocs.length; if (numberOfResults > offset) { // There are enough results to satisfy the offset ... - PathFactory pathFactory = context.getValueFactories().getPathFactory(); for (int i = offset, num = scoreDocs.length; i != num; ++i) { ScoreDoc result = scoreDocs[i]; int docId = result.doc; // Find the UUID of the node (this UUID might be artificial, so we have to find the path) ... Document doc = contentReader.document(docId, UUID_FIELD_SELECTOR); - String uuid = doc.get(ContentIndex.UUID); - // Find the path for this node (is there a better way to do this than one search per UUID?) ... - TopDocs pathDocs = pathSearcher.search(new TermQuery(new Term(PathIndex.UUID, uuid)), 1); - if (pathDocs.scoreDocs.length < 1) { + String id = doc.get(ContentIndex.ID); + Location location = getLocationForDocument(id, pathReader, pathSearcher); + if (location == null) { // No path record found ... continue; } - Document pathDoc = pathReader.document(pathDocs.scoreDocs[0].doc); - String pathString = pathDoc.get(PathIndex.PATH); - Path path = pathFactory.create(pathString); // Now add the location ... - results.add(Location.create(path, UUID.fromString(uuid))); + results.add(location); } } } catch (ParseException e) { @@ -579,6 +671,26 @@ } } + protected Location getLocationForDocument( String id, + IndexReader pathReader, + IndexSearcher pathSearcher ) throws IOException { + // Find the path for this node (is there a better way to do this than one search per ID?) ... + TopDocs pathDocs = pathSearcher.search(new TermQuery(new Term(PathIndex.ID, id)), 1); + if (pathDocs.scoreDocs.length < 1) { + // No path record found ... + return null; + } + Document pathDoc = pathReader.document(pathDocs.scoreDocs[0].doc); + return readLocation(pathDoc); + } + + protected UUID getUuid( Document document, + Name name ) { + String nameString = stringFactory.create(name); + String value = document.get(nameString); + return value != null ? uuidFactory.create(value) : null; + } + /** * {@inheritDoc} * @@ -802,13 +914,23 @@ } /** - * Get the set of UUIDs for the children of the node at the given path. + * {@inheritDoc} * + * @see org.jboss.dna.search.LuceneSession#createTupleCollector(Columns) + */ + @Override + public TupleCollector createTupleCollector( Columns columns ) { + return new DualIndexTupleCollector(this, columns); + } + + /** + * Get the set of IDs for the children of the node at the given path. + * * @param parentPath the path to the parent node; may not be null - * @return the UUIDs of the child nodes; never null but possibly empty + * @return the doc IDs of the child nodes; never null but possibly empty * @throws IOException if there is an error accessing the indexes */ - protected Set<UUID> getUuidsForChildrenOf( Path parentPath ) throws IOException { + protected Set<String> getIdsForChildrenOf( Path parentPath ) throws IOException { // Find the path of the parent ... String stringifiedPath = pathAsString(parentPath, stringFactory); // Append a '/' to the parent path, so we'll only get decendants ... @@ -825,23 +947,23 @@ combinedQuery.add(depthQuery, Occur.MUST); query = combinedQuery; - // Now execute and collect the UUIDs ... - UuidCollector uuidCollector = new UuidCollector(); + // Now execute and collect the IDs ... + IdCollector idCollector = new IdCollector(); IndexSearcher searcher = getPathsSearcher(); - searcher.search(query, uuidCollector); - return uuidCollector.getUuids(); + searcher.search(query, idCollector); + return idCollector.getIds(); } /** - * Get the set of UUIDs for the nodes that are descendants of the node at the given path. + * Get the set of IDs for the nodes that are descendants of the node at the given path. * * @param parentPath the path to the parent node; may not be null and <i>may not be the root node</i> * @param includeParent true if the parent node should be included in the results, or false if only the descendants should * be included - * @return the UUIDs of the nodes; never null but possibly empty + * @return the IDs of the nodes; never null but possibly empty * @throws IOException if there is an error accessing the indexes */ - protected Set<UUID> getUuidsForDescendantsOf( Path parentPath, + protected Set<String> getIdsForDescendantsOf( Path parentPath, boolean includeParent ) throws IOException { assert !parentPath.isRoot(); @@ -855,21 +977,21 @@ // Create a prefix query ... Query query = new PrefixQuery(new Term(PathIndex.PATH, stringifiedPath)); - // Now execute and collect the UUIDs ... - UuidCollector uuidCollector = new UuidCollector(); + // Now execute and collect the IDs ... + IdCollector idCollector = new IdCollector(); IndexSearcher searcher = getPathsSearcher(); - searcher.search(query, uuidCollector); - return uuidCollector.getUuids(); + searcher.search(query, idCollector); + return idCollector.getIds(); } /** - * Get the set containing the single UUID for the node at the given path. + * Get the set containing the single ID for the node at the given path. * * @param path the path to the node; may not be null - * @return the UUID of the supplied node; or null if the node cannot be found + * @return the ID of the supplied node; or null if the node cannot be found * @throws IOException if there is an error accessing the indexes */ - protected UUID getUuidFor( Path path ) throws IOException { + protected String getIdFor( Path path ) throws IOException { // Create a query to find all the nodes below the parent path ... IndexSearcher searcher = getPathsSearcher(); String stringifiedPath = pathAsString(path, stringFactory); @@ -879,38 +1001,38 @@ TopDocs topDocs = searcher.search(query, 1); if (topDocs.totalHits == 0) return null; Document pathDoc = getPathsReader().document(topDocs.scoreDocs[0].doc); - String uuidString = pathDoc.get(PathIndex.UUID); - return UUID.fromString(uuidString); + String idString = pathDoc.get(PathIndex.ID); + assert idString != null; + return idString; } /** - * Utility method to create a query to find all of the documents representing nodes with the supplied UUIDs. + * {@inheritDoc} * - * @param uuids the UUIDs of the nodes that are to be found; may not be null - * @return the query; never null + * @see org.jboss.dna.search.LuceneSession#findAllNodesWithIds(java.util.Set) */ @Override - public Query findAllNodesWithUuids( Set<UUID> uuids ) { - if (uuids.isEmpty()) { + public Query findAllNodesWithIds( Set<String> ids ) { + if (ids.isEmpty()) { // There are no children, so return a null query ... return new MatchNoneQuery(); } - if (uuids.size() == 1) { - UUID uuid = uuids.iterator().next(); - if (uuid == null) return new MatchNoneQuery(); - return new TermQuery(new Term(ContentIndex.UUID, uuid.toString())); + if (ids.size() == 1) { + String id = ids.iterator().next(); + if (id == null) return new MatchNoneQuery(); + return new TermQuery(new Term(ContentIndex.ID, id)); } - if (uuids.size() < 50) { + if (ids.size() < 50) { // Create an OR boolean query for all the UUIDs, since this is probably more efficient ... BooleanQuery query = new BooleanQuery(); - for (UUID uuid : uuids) { - Query uuidQuery = new TermQuery(new Term(ContentIndex.UUID, uuid.toString())); + for (String id : ids) { + Query uuidQuery = new TermQuery(new Term(ContentIndex.ID, id)); query.add(uuidQuery, Occur.SHOULD); } return query; } // Return a query that will always find all of the UUIDs ... - return new UuidsQuery(ContentIndex.UUID, uuids, getContext().getValueFactories().getUuidFactory()); + return new IdsQuery(ContentIndex.ID, ids); } @Override @@ -918,8 +1040,8 @@ if (ancestorPath.isRoot()) { return new MatchAllDocsQuery(); } - Set<UUID> uuids = getUuidsForDescendantsOf(ancestorPath, false); - return findAllNodesWithUuids(uuids); + Set<String> ids = getIdsForDescendantsOf(ancestorPath, false); + return findAllNodesWithIds(ids); } /** @@ -935,58 +1057,37 @@ if (parentPath.isRoot()) { return new MatchAllDocsQuery(); } - Set<UUID> childUuids = getUuidsForChildrenOf(parentPath); - return findAllNodesWithUuids(childUuids); + Set<String> childIds = getIdsForChildrenOf(parentPath); + return findAllNodesWithIds(childIds); } /** * Create a query that can be used to find the one document (or node) that exists at the exact path supplied. This method - * first queries the {@link PathIndex path index} to find the UUID of the node at the supplied path, and then returns a - * query that matches the UUID. + * first queries the {@link PathIndex path index} to find the ID of the node at the supplied path, and then returns a + * query that matches the ID. * * @param path the path of the node * @return the query; never null - * @throws IOException if there is an error finding the UUID for the supplied path + * @throws IOException if there is an error finding the ID for the supplied path */ @Override public Query findNodeAt( Path path ) throws IOException { - UUID uuid = getUuidFor(path); - if (uuid == null) return null; - return new TermQuery(new Term(ContentIndex.UUID, uuid.toString())); + String id = getIdFor(path); + if (id == null) return null; + return new TermQuery(new Term(ContentIndex.ID, id)); } /** - * Create a query that can be used to find documents (or nodes) that have a field value that satisfies the supplied LIKE - * expression. + * {@inheritDoc} * - * @param fieldName the name of the document field to search - * @param likeExpression the JCR like expression - * @return the query; never null + * @see org.jboss.dna.search.LuceneSession#findNodesLike(java.lang.String, java.lang.String, boolean) */ @Override public Query findNodesLike( String fieldName, - String likeExpression ) { - assert likeExpression != null; - assert likeExpression.length() > 0; - - // '%' matches 0 or more characters - // '_' matches any single character - // '\x' matches 'x' - // all other characters match themselves - - // Wildcard queries are a better match, but they can be slow and should not be used - // if the first character of the expression is a '%' or '_' ... - char firstChar = likeExpression.charAt(0); - if (firstChar != '%' && firstChar != '_') { - // Create a wildcard query ... - String expression = toWildcardExpression(likeExpression); - return new WildcardQuery(new Term(fieldName, expression)); - } - // Create a regex query, - String regex = toRegularExpression(likeExpression); - RegexQuery query = new RegexQuery(new Term(fieldName, regex)); - query.setRegexImplementation(new JavaUtilRegexCapabilities()); - return query; + String likeExpression, + boolean caseSensitive ) { + ValueFactories factories = context.getValueFactories(); + return CompareStringQuery.createQueryForNodesWithFieldLike(likeExpression, fieldName, factories, caseSensitive); } @Override @@ -1042,9 +1143,15 @@ if (!caseSensitive) stringValue = stringValue.toLowerCase(); switch (operator) { case EQUAL_TO: - return new TermQuery(new Term(field, stringValue)); + return CompareStringQuery.createQueryForNodesWithFieldEqualTo(stringValue, + field, + factories, + caseSensitive); case NOT_EQUAL_TO: - Query query = new TermQuery(new Term(field, stringValue)); + Query query = CompareStringQuery.createQueryForNodesWithFieldEqualTo(stringValue, + field, + factories, + caseSensitive); return new NotQuery(query); case GREATER_THAN: return CompareStringQuery.createQueryForNodesWithFieldGreaterThan(stringValue, @@ -1067,7 +1174,7 @@ factories, caseSensitive); case LIKE: - return findNodesLike(field, stringValue); + return findNodesLike(field, stringValue, caseSensitive); } break; case DATE: @@ -1239,7 +1346,8 @@ return new NotQuery(findNodeAt(pathValue)); case LIKE: String likeExpression = stringFactory.create(value); - return findNodesLike(PathIndex.PATH, likeExpression); + query = findNodesLike(PathIndex.PATH, likeExpression, caseSensitive); + break; case GREATER_THAN: query = ComparePathQuery.createQueryForNodesWithPathGreaterThan(pathValue, PathIndex.PATH, @@ -1265,11 +1373,11 @@ caseSensitive); break; } - // Now execute and collect the UUIDs ... - UuidCollector uuidCollector = new UuidCollector(); + // Now execute and collect the IDs ... + IdCollector idCollector = new IdCollector(); IndexSearcher searcher = getPathsSearcher(); - searcher.search(query, uuidCollector); - return findAllNodesWithUuids(uuidCollector.getUuids()); + searcher.search(query, idCollector); + return findAllNodesWithIds(idCollector.getIds()); } @Override @@ -1277,6 +1385,7 @@ Operator operator, Object value, boolean caseSensitive ) throws IOException { + ValueFactories factories = getContext().getValueFactories(); String stringValue = stringFactory.create(value); if (!caseSensitive) stringValue = stringValue.toLowerCase(); Path.Segment segment = operator != Operator.LIKE ? pathFactory.createSegment(stringValue) : null; @@ -1285,42 +1394,42 @@ switch (operator) { case EQUAL_TO: BooleanQuery booleanQuery = new BooleanQuery(); - booleanQuery.add(new TermQuery(new Term(PathIndex.LOCAL_NAME, stringValue)), Occur.MUST); + booleanQuery.add(new TermQuery(new Term(PathIndex.NODE_NAME, stringValue)), Occur.MUST); booleanQuery.add(NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, snsIndex, snsIndex, true, false), Occur.MUST); return booleanQuery; case NOT_EQUAL_TO: booleanQuery = new BooleanQuery(); - booleanQuery.add(new TermQuery(new Term(PathIndex.LOCAL_NAME, stringValue)), Occur.MUST); + booleanQuery.add(new TermQuery(new Term(PathIndex.NODE_NAME, stringValue)), Occur.MUST); booleanQuery.add(NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, snsIndex, snsIndex, true, false), Occur.MUST); return new NotQuery(booleanQuery); case GREATER_THAN: query = CompareNameQuery.createQueryForNodesWithNameGreaterThan(segment, - PathIndex.LOCAL_NAME, + PathIndex.NODE_NAME, PathIndex.SNS_INDEX, - context.getValueFactories(), + factories, caseSensitive); break; case GREATER_THAN_OR_EQUAL_TO: query = CompareNameQuery.createQueryForNodesWithNameGreaterThanOrEqualTo(segment, - PathIndex.LOCAL_NAME, + PathIndex.NODE_NAME, PathIndex.SNS_INDEX, - context.getValueFactories(), + factories, caseSensitive); break; case LESS_THAN: query = CompareNameQuery.createQueryForNodesWithNameLessThan(segment, - PathIndex.LOCAL_NAME, + PathIndex.NODE_NAME, PathIndex.SNS_INDEX, - context.getValueFactories(), + factories, caseSensitive); break; case LESS_THAN_OR_EQUAL_TO: query = CompareNameQuery.createQueryForNodesWithNameLessThanOrEqualTo(segment, - PathIndex.LOCAL_NAME, + PathIndex.NODE_NAME, PathIndex.SNS_INDEX, - context.getValueFactories(), + factories, caseSensitive); break; case LIKE: @@ -1330,7 +1439,10 @@ if (openBracketIndex != -1) { String localNameExpression = likeExpression.substring(0, openBracketIndex); String snsIndexExpression = likeExpression.substring(openBracketIndex); - Query localNameQuery = createLocalNameQuery(localNameExpression); + Query localNameQuery = CompareStringQuery.createQueryForNodesWithFieldLike(localNameExpression, + PathIndex.NODE_NAME, + factories, + caseSensitive); Query snsQuery = createSnsIndexQuery(snsIndexExpression); if (localNameQuery == null) { if (snsQuery == null) { @@ -1353,17 +1465,20 @@ } } else { // There is no SNS expression ... - query = createLocalNameQuery(likeExpression); + query = CompareStringQuery.createQueryForNodesWithFieldLike(likeExpression, + PathIndex.NODE_NAME, + factories, + caseSensitive); } assert query != null; break; } - // Now execute and collect the UUIDs ... - UuidCollector uuidCollector = new UuidCollector(); + // Now execute and collect the IDs ... + IdCollector idCollector = new IdCollector(); IndexSearcher searcher = getPathsSearcher(); - searcher.search(query, uuidCollector); - return findAllNodesWithUuids(uuidCollector.getUuids()); + searcher.search(query, idCollector); + return findAllNodesWithIds(idCollector.getIds()); } @Override @@ -1376,12 +1491,20 @@ switch (operator) { case LIKE: String likeExpression = stringFactory.create(value); - return findNodesLike(PathIndex.LOCAL_NAME, likeExpression); // already is a query with UUIDs + query = findNodesLike(PathIndex.LOCAL_NAME, likeExpression, caseSensitive); + break; case EQUAL_TO: - query = new TermQuery(new Term(PathIndex.LOCAL_NAME, nameValue)); + query = CompareStringQuery.createQueryForNodesWithFieldEqualTo(nameValue, + PathIndex.LOCAL_NAME, + context.getValueFactories(), + caseSensitive); break; case NOT_EQUAL_TO: - query = new NotQuery(new TermQuery(new Term(PathIndex.LOCAL_NAME, nameValue))); + query = CompareStringQuery.createQueryForNodesWithFieldEqualTo(nameValue, + PathIndex.LOCAL_NAME, + context.getValueFactories(), + caseSensitive); + query = new NotQuery(query); break; case GREATER_THAN: query = CompareStringQuery.createQueryForNodesWithFieldGreaterThan(nameValue, @@ -1409,11 +1532,11 @@ break; } - // Now execute and collect the UUIDs ... - UuidCollector uuidCollector = new UuidCollector(); + // Now execute and collect the IDs ... + IdCollector idCollector = new IdCollector(); IndexSearcher searcher = getPathsSearcher(); - searcher.search(query, uuidCollector); - return findAllNodesWithUuids(uuidCollector.getUuids()); + searcher.search(query, idCollector); + return findAllNodesWithIds(idCollector.getIds()); } @Override @@ -1447,25 +1570,34 @@ return null; } - // Now execute and collect the UUIDs ... - UuidCollector uuidCollector = new UuidCollector(); + // Now execute and collect the IDs ... + IdCollector idCollector = new IdCollector(); IndexSearcher searcher = getPathsSearcher(); - searcher.search(query, uuidCollector); - return findAllNodesWithUuids(uuidCollector.getUuids()); + searcher.search(query, idCollector); + return findAllNodesWithIds(idCollector.getIds()); } - protected Query createLocalNameQuery( String likeExpression ) { + protected Query createLocalNameQuery( String likeExpression, + boolean caseSensitive ) { if (likeExpression == null) return null; - likeExpression = likeExpression.trim(); - if (likeExpression.length() == 0) return null; - if (likeExpression.indexOf('?') != -1 || likeExpression.indexOf('*') != -1) { - // The local name is a like ... - return findNodesLike(PathIndex.LOCAL_NAME, likeExpression); - } - // The local name is an exact match ... - return new TermQuery(new Term(PathIndex.LOCAL_NAME, likeExpression)); + ValueFactories factories = getContext().getValueFactories(); + return CompareStringQuery.createQueryForNodesWithFieldLike(likeExpression, + PathIndex.LOCAL_NAME, + factories, + caseSensitive); } + /** + * Utility method to generate a query against the SNS indexes. This method attempts to generate a query that works most + * efficiently, depending upon the supplied expression. For example, if the supplied expression is just "[3]", then a + * range query is used to find all values matching '3'. However, if "[3_]" is used (where '_' matches any + * single-character, or digit in this case), then a range query is used to find all values between '30' and '39'. + * Similarly, if "[3%]" is used, then a regular expression query is used. + * + * @param likeExpression the expression that uses the JCR 2.0 LIKE representation, and which includes the leading '[' and + * trailing ']' characters + * @return the query, or null if the expression cannot be represented as a query + */ protected Query createSnsIndexQuery( String likeExpression ) { if (likeExpression == null) return null; likeExpression = likeExpression.trim(); @@ -1480,16 +1612,51 @@ if (closeBracketIndex != -1) { likeExpression = likeExpression.substring(0, closeBracketIndex); } - // If SNS expression contains '?' or '*' ... - if (likeExpression.indexOf('?') != -1 || likeExpression.indexOf('*') != -1) { - // There is a LIKE expression for the SNS ... - return findNodesLike(PathIndex.SNS_INDEX, likeExpression); + if (likeExpression.equals("_")) { + // The SNS expression can only be one digit ... + return NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, MIN_SNS_INDEX, 9, true, true); } + if (likeExpression.equals("%")) { + // The SNS expression can be any digits ... + return NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, MIN_SNS_INDEX, MAX_SNS_INDEX, true, true); + } + if (likeExpression.indexOf('_') != -1) { + if (likeExpression.indexOf('%') != -1) { + // Contains both ... + return findNodesLike(PathIndex.SNS_INDEX, likeExpression, true); + } + // It presumably contains some numbers and at least one '_' character ... + int firstWildcardChar = likeExpression.indexOf('_'); + if (firstWildcardChar + 1 < likeExpression.length()) { + // There's at least some characters after the first '_' ... + int secondWildcardChar = likeExpression.indexOf('_', firstWildcardChar + 1); + if (secondWildcardChar != -1) { + // There are multiple '_' characters ... + return findNodesLike(PathIndex.SNS_INDEX, likeExpression, true); + } + } + // There's only one '_', so parse the lowermost value and uppermost value ... + String lowerExpression = likeExpression.replace('_', '0'); + String upperExpression = likeExpression.replace('_', '9'); + try { + // This SNS is just a number ... + int lowerSns = Integer.parseInt(lowerExpression); + int upperSns = Integer.parseInt(upperExpression); + return NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, lowerSns, upperSns, true, true); + } catch (NumberFormatException e) { + // It's not a number but it's in the SNS field, so there will be no results ... + return new MatchNoneQuery(); + } + } + if (likeExpression.indexOf('%') != -1) { + // It presumably contains some numbers and at least one '%' character ... + return findNodesLike(PathIndex.SNS_INDEX, likeExpression, true); + } // This is not a LIKE expression but an exact value specification and should be a number ... try { // This SNS is just a number ... int sns = Integer.parseInt(likeExpression); - return NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, sns, sns, true, false); + return NumericRangeQuery.newIntRange(PathIndex.SNS_INDEX, sns, sns, true, true); } catch (NumberFormatException e) { // It's not a number but it's in the SNS field, so there will be no results ... return new MatchNoneQuery(); @@ -1505,13 +1672,13 @@ * * @see DualIndexSearchProvider.DualIndexSession#findChildNodes(Path) */ - protected static class UuidCollector extends Collector { - private final Set<UUID> uuids = new HashSet<UUID>(); - private String[] uuidsByDocId; + protected static class IdCollector extends Collector { + private final Set<String> ids = new HashSet<String>(); + private String[] idsByDocId; // private int baseDocId; - protected UuidCollector() { + protected IdCollector() { } /** @@ -1519,8 +1686,8 @@ * * @return the set of UUIDs; never null */ - public Set<UUID> getUuids() { - return uuids; + public Set<String> getIds() { + return ids; } /** @@ -1551,9 +1718,9 @@ @Override public void collect( int docId ) { assert docId >= 0; - String uuidString = uuidsByDocId[docId]; - assert uuidString != null; - uuids.add(UUID.fromString(uuidString)); + String idString = idsByDocId[docId]; + assert idString != null; + ids.add(idString); } /** @@ -1564,8 +1731,148 @@ @Override public void setNextReader( IndexReader reader, int docBase ) throws IOException { - this.uuidsByDocId = FieldCache.DEFAULT.getStrings(reader, UUID_FIELD); + this.idsByDocId = FieldCache.DEFAULT.getStrings(reader, ContentIndex.ID); // same value as PathIndex.ID // this.baseDocId = docBase; } } + + /** + * This collector is responsible for loading the value for each of the columns into each tuple array. + */ + protected class DualIndexTupleCollector extends TupleCollector { + private final DualIndexSession session; + private final LinkedList<Object[]> tuples = new LinkedList<Object[]>(); + private final Columns columns; + private final int numValues; + private final boolean recordScore; + private final int scoreIndex; + private final FieldSelector fieldSelector; + private final int locationIndex; + private Scorer scorer; + private IndexReader currentReader; + private int docOffset; + private boolean resolvedLocations = false; + + protected DualIndexTupleCollector( DualIndexSession session, + Columns columns ) { + this.session = session; + this.columns = columns; + assert this.session != null; + assert this.columns != null; + this.numValues = this.columns.getTupleSize(); + assert this.numValues >= 0; + assert this.columns.getSelectorNames().size() == 1; + final String selectorName = this.columns.getSelectorNames().get(0); + this.locationIndex = this.columns.getLocationIndex(selectorName); + this.recordScore = this.columns.hasFullTextSearchScores(); + this.scoreIndex = this.recordScore ? this.columns.getFullTextSearchScoreIndexFor(selectorName) : -1; + final Set<String> columnNames = new HashSet<String>(this.columns.getColumnNames()); + columnNames.add(ContentIndex.ID); // add the UUID, which we'll put into the Location ... + this.fieldSelector = new FieldSelector() { + private static final long serialVersionUID = 1L; + + public FieldSelectorResult accept( String fieldName ) { + return columnNames.contains(fieldName) ? FieldSelectorResult.LOAD : FieldSelectorResult.NO_LOAD; + } + }; + } + + /** + * @return tuples + */ + @Override + public LinkedList<Object[]> getTuples() { + resolveLocations(); + return tuples; + } + + protected void resolveLocations() { + if (resolvedLocations) return; + try { + // The Location field in the tuples all contain the ID of the document, so we need to replace these + // with the appropriate Location objects, using the content from the PathIndex ... + IndexReader pathReader = session.getPathsReader(); + IndexSearcher pathSearcher = session.getPathsSearcher(); + for (Object[] tuple : tuples) { + String id = (String)tuple[locationIndex]; + assert id != null; + Location location = session.getLocationForDocument(id, pathReader, pathSearcher); + assert location != null; + tuple[locationIndex] = location; + } + resolvedLocations = true; + } catch (IOException e) { + throw new LuceneException(e); + } + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Collector#acceptsDocsOutOfOrder() + */ + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + /** + * Get the location index. + * + * @return locationIndex + */ + public int getLocationIndex() { + return locationIndex; + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Collector#setNextReader(org.apache.lucene.index.IndexReader, int) + */ + @Override + public void setNextReader( IndexReader reader, + int docBase ) { + this.currentReader = reader; + this.docOffset = docBase; + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Collector#setScorer(org.apache.lucene.search.Scorer) + */ + @Override + public void setScorer( Scorer scorer ) { + this.scorer = scorer; + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Collector#collect(int) + */ + @Override + public void collect( int doc ) throws IOException { + int docId = doc + docOffset; + Object[] tuple = new Object[numValues]; + Document document = currentReader.document(docId, fieldSelector); + for (String columnName : columns.getColumnNames()) { + int index = columns.getColumnIndexForName(columnName); + // We just need to retrieve the first value if there is more than one ... + tuple[index] = document.get(columnName); + } + + // Set the score column if required ... + if (recordScore) { + assert scorer != null; + tuple[scoreIndex] = scorer.score(); + } + + // Load the document ID (which is a stringified UUID) into the Location slot, + // which will be replaced later with a real Location ... + tuple[locationIndex] = document.get(ContentIndex.ID); + tuples.add(tuple); + } + } } Modified: trunk/dna-search/src/main/java/org/jboss/dna/search/IndexRules.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/IndexRules.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/IndexRules.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -339,7 +339,7 @@ */ public static Builder createBuilder( IndexRules initialRules ) { CheckArg.isNotNull(initialRules, "initialRules"); - return new Builder(initialRules.rulesByName).defaultTo(initialRules.defaultRule); + return new Builder(new HashMap<Name, Rule>(initialRules.rulesByName)).defaultTo(initialRules.defaultRule); } /** Modified: trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneQueryComponent.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneQueryComponent.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneQueryComponent.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -24,28 +24,17 @@ package org.jboss.dna.search; import java.io.IOException; -import java.util.HashSet; -import java.util.LinkedList; import java.util.List; -import java.util.Set; -import java.util.UUID; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.document.FieldSelectorResult; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.jboss.dna.common.i18n.I18n; import org.jboss.dna.graph.ExecutionContext; -import org.jboss.dna.graph.Location; import org.jboss.dna.graph.property.Binary; import org.jboss.dna.graph.property.Name; import org.jboss.dna.graph.property.PropertyType; @@ -88,6 +77,7 @@ import org.jboss.dna.graph.query.process.SelectComponent; import org.jboss.dna.graph.query.process.SelectComponent.Analyzer; import org.jboss.dna.search.DualIndexSearchProvider.ContentIndex; +import org.jboss.dna.search.LuceneSession.TupleCollector; /** * @@ -210,7 +200,7 @@ try { // Execute the query against the content indexes ... IndexSearcher searcher = session.getContentSearcher(); - TupleCollector collector = new TupleCollector(columns, execContext.getValueFactories().getUuidFactory()); + TupleCollector collector = session.createTupleCollector(columns); searcher.search(pushDownQuery, collector); tuples = collector.getTuples(); } catch (IOException e) { @@ -498,112 +488,4 @@ assert false; return null; } - - /** - * This collector is responsible for loading the value for each of the columns into each tuple array. - */ - protected static class TupleCollector extends Collector { - private final LinkedList<Object[]> tuples = new LinkedList<Object[]>(); - private final Columns columns; - private final int numValues; - private final boolean recordScore; - private final int scoreIndex; - private final FieldSelector fieldSelector; - private final int locationIndex; - private final ValueFactory<UUID> uuidFactory; - private Scorer scorer; - private IndexReader currentReader; - private int docOffset; - - protected TupleCollector( Columns columns, - ValueFactory<UUID> uuidFactory ) { - this.columns = columns; - this.uuidFactory = uuidFactory; - assert this.columns != null; - assert this.uuidFactory != null; - this.numValues = this.columns.getTupleSize(); - assert this.numValues >= 0; - assert this.columns.getSelectorNames().size() == 1; - final String selectorName = this.columns.getSelectorNames().get(0); - this.locationIndex = this.columns.getLocationIndex(selectorName); - this.recordScore = this.columns.hasFullTextSearchScores(); - this.scoreIndex = this.recordScore ? this.columns.getFullTextSearchScoreIndexFor(selectorName) : -1; - final Set<String> columnNames = new HashSet<String>(this.columns.getColumnNames()); - columnNames.add(ContentIndex.UUID); // add the UUID, which we'll put into the Location ... - this.fieldSelector = new FieldSelector() { - private static final long serialVersionUID = 1L; - - public FieldSelectorResult accept( String fieldName ) { - return columnNames.contains(fieldName) ? FieldSelectorResult.LOAD : FieldSelectorResult.NO_LOAD; - } - }; - } - - /** - * @return tuples - */ - public LinkedList<Object[]> getTuples() { - return tuples; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Collector#acceptsDocsOutOfOrder() - */ - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Collector#setNextReader(org.apache.lucene.index.IndexReader, int) - */ - @Override - public void setNextReader( IndexReader reader, - int docBase ) { - this.currentReader = reader; - this.docOffset = docBase; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Collector#setScorer(org.apache.lucene.search.Scorer) - */ - @Override - public void setScorer( Scorer scorer ) { - this.scorer = scorer; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Collector#collect(int) - */ - @Override - public void collect( int doc ) throws IOException { - int docId = doc + docOffset; - Object[] tuple = new Object[numValues]; - Document document = currentReader.document(docId, fieldSelector); - for (String columnName : columns.getColumnNames()) { - int index = columns.getColumnIndexForName(columnName); - // We just need to retrieve the first value if there is more than one ... - tuple[index] = document.get(columnName); - } - - // Set the score column if required ... - if (recordScore) { - assert scorer != null; - tuple[scoreIndex] = scorer.score(); - } - - // Load the UUID into a Location object ... - UUID uuid = uuidFactory.create(document.get(ContentIndex.UUID)); - tuple[locationIndex] = Location.create(uuid); - tuples.add(tuple); - } - } } Modified: trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneSession.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneSession.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/LuceneSession.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -26,16 +26,19 @@ import java.io.IOException; import java.util.LinkedList; import java.util.Set; -import java.util.UUID; import net.jcip.annotations.NotThreadSafe; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.jboss.dna.common.i18n.I18n; import org.jboss.dna.graph.ExecutionContext; import org.jboss.dna.graph.property.DateTimeFactory; +import org.jboss.dna.graph.property.NameFactory; import org.jboss.dna.graph.property.Path; import org.jboss.dna.graph.property.PathFactory; +import org.jboss.dna.graph.property.UuidFactory; +import org.jboss.dna.graph.property.ValueFactories; import org.jboss.dna.graph.property.ValueFactory; import org.jboss.dna.graph.query.QueryContext; import org.jboss.dna.graph.query.QueryEngine; @@ -76,6 +79,8 @@ protected final ValueFactory<String> stringFactory; protected final DateTimeFactory dateFactory; protected final PathFactory pathFactory; + protected final UuidFactory uuidFactory; + protected final NameFactory nameFactory; private int changeCount; private QueryEngine queryEngine; @@ -93,9 +98,12 @@ this.overwrite = overwrite; this.readOnly = readOnly; this.analyzer = analyzer; - this.stringFactory = context.getValueFactories().getStringFactory(); - this.dateFactory = context.getValueFactories().getDateFactory(); - this.pathFactory = context.getValueFactories().getPathFactory(); + ValueFactories factories = context.getValueFactories(); + this.stringFactory = factories.getStringFactory(); + this.dateFactory = factories.getDateFactory(); + this.pathFactory = factories.getPathFactory(); + this.uuidFactory = factories.getUuidFactory(); + this.nameFactory = factories.getNameFactory(); assert this.context != null; assert this.sourceName != null; assert this.workspaceName != null; @@ -227,13 +235,21 @@ throws IOException; /** - * Utility method to create a query to find all of the documents representing nodes with the supplied UUIDs. + * Create a {@link TupleCollector} instance that collects the results from the index(es). * - * @param uuids the UUIDs of the nodes that are to be found; may not be null + * @param columns the column definitions; never null + * @return the collector; never null + */ + public abstract TupleCollector createTupleCollector( Columns columns ); + + /** + * Utility method to create a query to find all of the documents representing nodes with the supplied IDs. + * + * @param ids the IDs of the nodes that are to be found; may not be null * @return the query; never null * @throws IOException if there is a problem creating this query */ - public abstract Query findAllNodesWithUuids( Set<UUID> uuids ) throws IOException; + public abstract Query findAllNodesWithIds( Set<String> ids ) throws IOException; public abstract Query findAllNodesBelow( Path ancestorPath ) throws IOException; @@ -264,11 +280,13 @@ * * @param fieldName the name of the document field to search * @param likeExpression the JCR like expression + * @param caseSensitive true if the evaluation should be performed in a case sensitive manner, or false otherwise * @return the query; never null * @throws IOException if there is an error creating the query */ public abstract Query findNodesLike( String fieldName, - String likeExpression ) throws IOException; + String likeExpression, + boolean caseSensitive ) throws IOException; public abstract Query findNodesWith( Length propertyLength, Operator operator, @@ -320,49 +338,28 @@ // public abstract Query createSnsIndexQuery( String likeExpression ) throws IOException; - /** - * Convert the JCR like expression to a Lucene wildcard expression. The JCR like expression uses '%' to match 0 or more - * characters, '_' to match any single character, '\x' to match the 'x' character, and all other characters to match - * themselves. - * - * @param likeExpression the like expression; may not be null - * @return the expression that can be used with a WildcardQuery; never null - */ - public String toWildcardExpression( String likeExpression ) { - assert likeExpression != null; - assert likeExpression.length() > 0; - return likeExpression.replace('%', '*').replace('_', '?').replaceAll("\\\\(.)", "$1"); - } - - /** - * Convert the JCR like expression to a regular expression. The JCR like expression uses '%' to match 0 or more characters, - * '_' to match any single character, '\x' to match the 'x' character, and all other characters to match themselves. Note that - * if any regex metacharacters appear in the like expression, they will be escaped within the resulting regular expression. - * - * @param likeExpression the like expression; may not be null - * @return the expression that can be used with a WildcardQuery; never null - */ - public String toRegularExpression( String likeExpression ) { - assert likeExpression != null; - assert likeExpression.length() > 0; - // Replace all '\x' with 'x' ... - String result = likeExpression.replaceAll("\\\\(.)", "$1"); - // Escape characters used as metacharacters in regular expressions, including - // '[', '^', '\', '$', '.', '|', '?', '*', '+', '(', and ')' - result = result.replaceAll("([[^\\\\$.|?*+()])", "\\$1"); - // Replace '%'->'[.]+' and '_'->'[.] - result = likeExpression.replace("%", "[.]+").replace("_", "[.]"); - return result; - } - public String pathAsString( Path path, ValueFactory<String> stringFactory ) { assert path != null; if (path.isRoot()) return "/"; - String pathStr = stringFactory.create(path); - if (!pathStr.endsWith("]")) { - pathStr = pathStr + '[' + Path.DEFAULT_INDEX + ']'; + StringBuilder sb = new StringBuilder(); + for (Path.Segment segment : path) { + sb.append('/'); + sb.append(stringFactory.create(segment.getName())); + sb.append('['); + sb.append(segment.getIndex()); + sb.append(']'); } - return pathStr; + return sb.toString(); } + + public static abstract class TupleCollector extends Collector { + + /** + * Get the tuples. + * + * @return the tuples; never null + */ + public abstract LinkedList<Object[]> getTuples(); + } } Modified: trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareQuery.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareQuery.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareQuery.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -96,7 +96,7 @@ * @param fieldSelector the field selector that should load the fields needed to recover the value; may be null if the field * selector should be generated automatically */ - protected CompareQuery( String fieldName, + protected CompareQuery( final String fieldName, ValueType constraintValue, ValueFactory<ValueType> valueTypeFactory, ValueFactory<String> stringFactory, @@ -114,7 +114,7 @@ private static final long serialVersionUID = 1L; public FieldSelectorResult accept( String fieldName ) { - return fieldName.equals(fieldName) ? FieldSelectorResult.LOAD_AND_BREAK : FieldSelectorResult.NO_LOAD; + return CompareQuery.this.fieldName.equals(fieldName) ? FieldSelectorResult.LOAD_AND_BREAK : FieldSelectorResult.NO_LOAD; } }; } @@ -133,7 +133,7 @@ */ @Override public Weight createWeight( Searcher searcher ) { - return new NotWeight(searcher); + return new CompareWeight(searcher); } /** @@ -149,11 +149,11 @@ /** * Calculates query weights and builds query scores for our NOT queries. */ - protected class NotWeight extends Weight { + protected class CompareWeight extends Weight { private static final long serialVersionUID = 1L; private final Searcher searcher; - protected NotWeight( Searcher searcher ) { + protected CompareWeight( Searcher searcher ) { this.searcher = searcher; assert this.searcher != null; } @@ -217,7 +217,7 @@ boolean scoreDocsInOrder, boolean topScorer ) { // Return a custom scorer ... - return new NotScorer(reader); + return new CompareScorer(reader); } /** @@ -235,17 +235,17 @@ /** * A scorer for the Path query. */ - protected class NotScorer extends Scorer { + protected class CompareScorer extends Scorer { private int docId = -1; private final int maxDocId; private final IndexReader reader; - protected NotScorer( IndexReader reader ) { + protected CompareScorer( IndexReader reader ) { // We don't care which Similarity we have, because we don't use it. So get the default. super(Similarity.getDefault()); this.reader = reader; assert this.reader != null; - this.maxDocId = this.reader.maxDoc(); + this.maxDocId = this.reader.maxDoc() - 1; } /** Modified: trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareStringQuery.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareStringQuery.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/query/CompareStringQuery.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -24,10 +24,16 @@ package org.jboss.dna.search.query; import java.io.IOException; +import java.util.regex.Pattern; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.search.regex.JavaUtilRegexCapabilities; +import org.apache.lucene.search.regex.RegexQuery; import org.jboss.dna.graph.property.ValueComparators; import org.jboss.dna.graph.property.ValueFactories; import org.jboss.dna.graph.property.ValueFactory; @@ -42,6 +48,19 @@ public class CompareStringQuery extends CompareQuery<String> { private static final long serialVersionUID = 1L; + protected static final Evaluator<String> EQUAL_TO = new Evaluator<String>() { + private static final long serialVersionUID = 1L; + + public boolean satisfiesConstraint( String nodeValue, + String constraintValue ) { + return constraintValue.equals(nodeValue); + } + + @Override + public String toString() { + return " = "; + } + }; protected static final Evaluator<String> IS_LESS_THAN = new Evaluator<String>() { private static final long serialVersionUID = 1L; @@ -96,6 +115,29 @@ }; /** + * Construct a {@link Query} implementation that scores documents with a string field value that is equal to the supplied + * constraint value. + * + * @param constraintValue the constraint value; may not be null + * @param fieldName the name of the document field containing the value; may not be null + * @param factories the value factories that can be used during the scoring; may not be null + * @param caseSensitive true if the comparison should be done in a case-sensitive manner, or false if it is to be + * case-insensitive + * @return the query; never null + */ + public static Query createQueryForNodesWithFieldEqualTo( String constraintValue, + String fieldName, + ValueFactories factories, + boolean caseSensitive ) { + if (caseSensitive) { + // We can just do a normal TermQuery ... + return new TermQuery(new Term(fieldName, constraintValue)); + } + return new CompareStringQuery(fieldName, constraintValue, factories.getStringFactory(), factories.getStringFactory(), + EQUAL_TO, caseSensitive); + } + + /** * Construct a {@link Query} implementation that scores documents with a string field value that is greater than the supplied * constraint value. * @@ -171,6 +213,83 @@ IS_LESS_THAN_OR_EQUAL_TO, caseSensitive); } + /** + * Construct a {@link Query} implementation that scores documents with a string field value that is LIKE the supplied + * constraint value. + * + * @param likeExpression the LIKE expression; may not be null + * @param fieldName the name of the document field containing the value; may not be null + * @param factories the value factories that can be used during the scoring; may not be null + * @param caseSensitive true if the comparison should be done in a case-sensitive manner, or false if it is to be + * case-insensitive + * @return the query; never null + */ + public static Query createQueryForNodesWithFieldLike( String likeExpression, + String fieldName, + ValueFactories factories, + boolean caseSensitive ) { + assert likeExpression != null; + assert likeExpression.length() > 0; + if (likeExpression.indexOf('%') == -1 && likeExpression.indexOf('_') == -1) { + // This is not a like expression, so just do an equals ... + return createQueryForNodesWithFieldEqualTo(likeExpression, fieldName, factories, caseSensitive); + } + if (caseSensitive) { + // We can just do a normal Wildcard or RegEx query ... + + // '%' matches 0 or more characters + // '_' matches any single character + // '\x' matches 'x' + // all other characters match themselves + + // Wildcard queries are a better match, but they can be slow and should not be used + // if the first character of the expression is a '%' or '_' ... + char firstChar = likeExpression.charAt(0); + if (firstChar != '%' && firstChar != '_') { + // Create a wildcard query ... + String expression = toWildcardExpression(likeExpression); + return new WildcardQuery(new Term(fieldName, expression)); + } + } + // Create a regex query (which will be done using the correct case) ... + String regex = toRegularExpression(likeExpression); + RegexQuery query = new RegexQuery(new Term(fieldName, regex)); + int flags = caseSensitive ? 0 : Pattern.CASE_INSENSITIVE; + query.setRegexImplementation(new JavaUtilRegexCapabilities(flags)); + return query; + } + + /** + * Convert the JCR like expression to a Lucene wildcard expression. The JCR like expression uses '%' to match 0 or more + * characters, '_' to match any single character, '\x' to match the 'x' character, and all other characters to match + * themselves. + * + * @param likeExpression the like expression; may not be null + * @return the expression that can be used with a WildcardQuery; never null + */ + protected static String toWildcardExpression( String likeExpression ) { + return likeExpression.replace('%', '*').replace('_', '?').replaceAll("\\\\(.)", "$1"); + } + + /** + * Convert the JCR like expression to a regular expression. The JCR like expression uses '%' to match 0 or more characters, + * '_' to match any single character, '\x' to match the 'x' character, and all other characters to match themselves. Note that + * if any regex metacharacters appear in the like expression, they will be escaped within the resulting regular expression. + * + * @param likeExpression the like expression; may not be null + * @return the expression that can be used with a WildcardQuery; never null + */ + protected static String toRegularExpression( String likeExpression ) { + // Replace all '\x' with 'x' ... + String result = likeExpression.replaceAll("\\\\(.)", "$1"); + // Escape characters used as metacharacters in regular expressions, including + // '[', '^', '\', '$', '.', '|', '?', '*', '+', '(', and ')' + result = result.replaceAll("([\\[^\\\\$.|?*+()])", "\\$1"); + // Replace '%'->'[.]+' and '_'->'[.] + result = likeExpression.replace("%", ".+").replace("_", "."); + return result; + } + private final boolean caseSensitive; /** @@ -204,6 +323,7 @@ protected String readFromDocument( IndexReader reader, int docId ) throws IOException { String result = super.readFromDocument(reader, docId); + if (result == null) return null; return caseSensitive ? result : result.toLowerCase(); } Copied: trunk/dna-search/src/main/java/org/jboss/dna/search/query/IdsQuery.java (from rev 1338, trunk/dna-search/src/main/java/org/jboss/dna/search/query/UuidsQuery.java) =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/query/IdsQuery.java (rev 0) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/query/IdsQuery.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -0,0 +1,261 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.dna.search.query; + +import java.io.IOException; +import java.util.Set; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.document.FieldSelectorResult; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.Weight; + +/** + * A Lucene {@link Query} implementation that is used to score positively those documents that have a ID in the supplied set. This + * works for large sets of IDs; in smaller numbers, it may be more efficient to create a boolean query that checks for each of the + * IDs. + */ +public class IdsQuery extends Query { + + private static final long serialVersionUID = 1L; + + /** + * The operand that is being negated by this query. + */ + protected final Set<String> uuids; + protected final FieldSelector fieldSelector; + protected final String fieldName; + + /** + * Construct a {@link Query} implementation that scores nodes according to the supplied comparator. + * + * @param fieldName the name of the document field containing the value; may not be null + * @param ids the set of ID values; may not be null + */ + public IdsQuery( String fieldName, + Set<String> ids ) { + this.fieldName = fieldName; + this.uuids = ids; + assert this.fieldName != null; + assert this.uuids != null; + this.fieldSelector = new FieldSelector() { + private static final long serialVersionUID = 1L; + + public FieldSelectorResult accept( String fieldName ) { + return fieldName.equals(fieldName) ? FieldSelectorResult.LOAD_AND_BREAK : FieldSelectorResult.NO_LOAD; + } + }; + } + + protected boolean includeDocument( IndexReader reader, + int docId ) throws IOException { + Document doc = reader.document(docId, fieldSelector); + String valueString = doc.get(fieldName); + return uuids.contains(valueString); + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Query#createWeight(org.apache.lucene.search.Searcher) + */ + @Override + public Weight createWeight( Searcher searcher ) { + return new IdSetWeight(searcher); + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Query#toString(java.lang.String) + */ + @Override + public String toString( String field ) { + return fieldName + " IN UUIDs"; + } + + /** + * Calculates query weights and builds query scores for our NOT queries. + */ + protected class IdSetWeight extends Weight { + private static final long serialVersionUID = 1L; + private final Searcher searcher; + + protected IdSetWeight( Searcher searcher ) { + this.searcher = searcher; + assert this.searcher != null; + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Weight#getQuery() + */ + @Override + public Query getQuery() { + return IdsQuery.this; + } + + /** + * {@inheritDoc} + * <p> + * This implementation always returns a weight factor of 1.0. + * </p> + * + * @see org.apache.lucene.search.Weight#getValue() + */ + @Override + public float getValue() { + return 1.0f; // weight factor of 1.0 + } + + /** + * {@inheritDoc} + * <p> + * This implementation always returns a normalization factor of 1.0. + * </p> + * + * @see org.apache.lucene.search.Weight#sumOfSquaredWeights() + */ + @Override + public float sumOfSquaredWeights() { + return 1.0f; // normalization factor of 1.0 + } + + /** + * {@inheritDoc} + * <p> + * This implementation always does nothing, as there is nothing to normalize. + * </p> + * + * @see org.apache.lucene.search.Weight#normalize(float) + */ + @Override + public void normalize( float norm ) { + // No need to do anything here + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Weight#scorer(org.apache.lucene.index.IndexReader, boolean, boolean) + */ + @Override + public Scorer scorer( IndexReader reader, + boolean scoreDocsInOrder, + boolean topScorer ) { + // Return a custom scorer ... + return new IdScorer(reader); + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.Weight#explain(org.apache.lucene.index.IndexReader, int) + */ + @Override + public Explanation explain( IndexReader reader, + int doc ) { + return new Explanation(getValue(), getQuery().toString()); + } + } + + /** + * A scorer for the Path query. + */ + protected class IdScorer extends Scorer { + private int docId = -1; + private final int maxDocId; + private final IndexReader reader; + + protected IdScorer( IndexReader reader ) { + // We don't care which Similarity we have, because we don't use it. So get the default. + super(Similarity.getDefault()); + this.reader = reader; + assert this.reader != null; + this.maxDocId = this.reader.maxDoc(); + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.DocIdSetIterator#docID() + */ + @Override + public int docID() { + return docId; + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.DocIdSetIterator#nextDoc() + */ + @Override + public int nextDoc() throws IOException { + do { + ++docId; + if (reader.isDeleted(docId)) { + // We should skip this document ... + continue; + } + if (includeDocument(reader, docId)) return docId; + } while (docId < maxDocId); + return Scorer.NO_MORE_DOCS; + } + + /** + * {@inheritDoc} + * + * @see org.apache.lucene.search.DocIdSetIterator#advance(int) + */ + @Override + public int advance( int target ) throws IOException { + if (target == Scorer.NO_MORE_DOCS) return target; + while (true) { + int doc = nextDoc(); + if (doc >= target) return doc; + } + } + + /** + * {@inheritDoc} + * <p> + * This method always returns a score of 1.0 for the current document, since only those documents that satisfy the NOT are + * scored by this scorer. + * </p> + * + * @see org.apache.lucene.search.Scorer#score() + */ + @Override + public float score() { + return 1.0f; + } + } +} Property changes on: trunk/dna-search/src/main/java/org/jboss/dna/search/query/IdsQuery.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Deleted: trunk/dna-search/src/main/java/org/jboss/dna/search/query/UuidsQuery.java =================================================================== --- trunk/dna-search/src/main/java/org/jboss/dna/search/query/UuidsQuery.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/main/java/org/jboss/dna/search/query/UuidsQuery.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -1,268 +0,0 @@ -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -package org.jboss.dna.search.query; - -import java.io.IOException; -import java.util.Set; -import java.util.UUID; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.document.FieldSelectorResult; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Searcher; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.Weight; -import org.jboss.dna.graph.property.ValueFactory; - -/** - * A Lucene {@link Query} implementation that is used to score positively those documents that have a UUID in the supplied set. - * This works for large sets of UUIDs; in smaller numbers, it may be more efficient to create a boolean query that checks for each - * fo the UUIDs. - */ -public class UuidsQuery extends Query { - - private static final long serialVersionUID = 1L; - - /** - * The operand that is being negated by this query. - */ - protected final Set<UUID> uuids; - protected final FieldSelector fieldSelector; - protected final String fieldName; - protected final ValueFactory<UUID> uuidFactory; - - /** - * Construct a {@link Query} implementation that scores nodes according to the supplied comparator. - * - * @param fieldName the name of the document field containing the value; may not be null - * @param uuids the set of UUID values; may not be null - * @param uuidFactory the factory to create UUID values; may not be null - */ - public UuidsQuery( String fieldName, - Set<UUID> uuids, - ValueFactory<UUID> uuidFactory ) { - this.fieldName = fieldName; - this.uuids = uuids; - this.uuidFactory = uuidFactory; - assert this.fieldName != null; - assert this.uuids != null; - assert this.uuidFactory != null; - this.fieldSelector = new FieldSelector() { - private static final long serialVersionUID = 1L; - - public FieldSelectorResult accept( String fieldName ) { - return fieldName.equals(fieldName) ? FieldSelectorResult.LOAD_AND_BREAK : FieldSelectorResult.NO_LOAD; - } - }; - } - - protected boolean includeDocument( IndexReader reader, - int docId ) throws IOException { - Document doc = reader.document(docId, fieldSelector); - String valueString = doc.get(fieldName); - return uuids.contains(uuidFactory.create(valueString)); - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Query#createWeight(org.apache.lucene.search.Searcher) - */ - @Override - public Weight createWeight( Searcher searcher ) { - return new UuidSetWeight(searcher); - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Query#toString(java.lang.String) - */ - @Override - public String toString( String field ) { - return fieldName + " IN UUIDs"; - } - - /** - * Calculates query weights and builds query scores for our NOT queries. - */ - protected class UuidSetWeight extends Weight { - private static final long serialVersionUID = 1L; - private final Searcher searcher; - - protected UuidSetWeight( Searcher searcher ) { - this.searcher = searcher; - assert this.searcher != null; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Weight#getQuery() - */ - @Override - public Query getQuery() { - return UuidsQuery.this; - } - - /** - * {@inheritDoc} - * <p> - * This implementation always returns a weight factor of 1.0. - * </p> - * - * @see org.apache.lucene.search.Weight#getValue() - */ - @Override - public float getValue() { - return 1.0f; // weight factor of 1.0 - } - - /** - * {@inheritDoc} - * <p> - * This implementation always returns a normalization factor of 1.0. - * </p> - * - * @see org.apache.lucene.search.Weight#sumOfSquaredWeights() - */ - @Override - public float sumOfSquaredWeights() { - return 1.0f; // normalization factor of 1.0 - } - - /** - * {@inheritDoc} - * <p> - * This implementation always does nothing, as there is nothing to normalize. - * </p> - * - * @see org.apache.lucene.search.Weight#normalize(float) - */ - @Override - public void normalize( float norm ) { - // No need to do anything here - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Weight#scorer(org.apache.lucene.index.IndexReader, boolean, boolean) - */ - @Override - public Scorer scorer( IndexReader reader, - boolean scoreDocsInOrder, - boolean topScorer ) { - // Return a custom scorer ... - return new UuidScorer(reader); - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.Weight#explain(org.apache.lucene.index.IndexReader, int) - */ - @Override - public Explanation explain( IndexReader reader, - int doc ) { - return new Explanation(getValue(), getQuery().toString()); - } - } - - /** - * A scorer for the Path query. - */ - protected class UuidScorer extends Scorer { - private int docId = -1; - private final int maxDocId; - private final IndexReader reader; - - protected UuidScorer( IndexReader reader ) { - // We don't care which Similarity we have, because we don't use it. So get the default. - super(Similarity.getDefault()); - this.reader = reader; - assert this.reader != null; - this.maxDocId = this.reader.maxDoc(); - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.DocIdSetIterator#docID() - */ - @Override - public int docID() { - return docId; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.DocIdSetIterator#nextDoc() - */ - @Override - public int nextDoc() throws IOException { - do { - ++docId; - if (reader.isDeleted(docId)) { - // We should skip this document ... - continue; - } - if (includeDocument(reader, docId)) return docId; - } while (docId < maxDocId); - return Scorer.NO_MORE_DOCS; - } - - /** - * {@inheritDoc} - * - * @see org.apache.lucene.search.DocIdSetIterator#advance(int) - */ - @Override - public int advance( int target ) throws IOException { - if (target == Scorer.NO_MORE_DOCS) return target; - while (true) { - int doc = nextDoc(); - if (doc >= target) return doc; - } - } - - /** - * {@inheritDoc} - * <p> - * This method always returns a score of 1.0 for the current document, since only those documents that satisfy the NOT are - * scored by this scorer. - * </p> - * - * @see org.apache.lucene.search.Scorer#score() - */ - @Override - public float score() { - return 1.0f; - } - } -} Modified: trunk/dna-search/src/test/java/org/jboss/dna/search/SearchEngineTest.java =================================================================== --- trunk/dna-search/src/test/java/org/jboss/dna/search/SearchEngineTest.java 2009-11-24 17:43:50 UTC (rev 1338) +++ trunk/dna-search/src/test/java/org/jboss/dna/search/SearchEngineTest.java 2009-11-24 17:44:29 UTC (rev 1339) @@ -26,6 +26,7 @@ import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; import java.io.IOException; import java.util.List; import org.jboss.dna.graph.ExecutionContext; @@ -35,10 +36,18 @@ import org.jboss.dna.graph.connector.RepositoryConnectionFactory; import org.jboss.dna.graph.connector.RepositorySourceException; import org.jboss.dna.graph.connector.inmemory.InMemoryRepositorySource; +import org.jboss.dna.graph.property.Name; import org.jboss.dna.graph.property.Path; +import org.jboss.dna.graph.query.QueryResults; +import org.jboss.dna.graph.query.model.QueryCommand; +import org.jboss.dna.graph.query.parse.SqlQueryParser; +import org.jboss.dna.graph.query.validate.ImmutableSchemata; +import org.jboss.dna.graph.query.validate.Schemata; +import org.jboss.dna.graph.query.validate.ImmutableSchemata.Builder; import org.jboss.dna.graph.search.SearchEngine; import org.jboss.dna.graph.search.SearchProvider; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.xml.sax.SAXException; @@ -53,6 +62,8 @@ private InMemoryRepositorySource source; private RepositoryConnectionFactory connectionFactory; private Graph content; + private Schemata schemata; + private SqlQueryParser sql; @Before public void beforeEach() throws Exception { @@ -79,14 +90,32 @@ }; // Set up the provider and the search engine ... - IndexRules rules = DualIndexSearchProvider.DEFAULT_RULES; + IndexRules.Builder rulesBuilder = IndexRules.createBuilder(DualIndexSearchProvider.DEFAULT_RULES); + // rulesBuilder.analyzeAndStoreAndFullText(name("maker")); + IndexRules rules = rulesBuilder.build(); LuceneConfiguration luceneConfig = LuceneConfigurations.inMemory(); // LuceneConfiguration luceneConfig = LuceneConfigurations.using(new File("target/testIndexes")); provider = new DualIndexSearchProvider(luceneConfig, rules); engine = new SearchEngine(context, sourceName, connectionFactory, provider); loadContent(); + + // Create the schemata for the workspaces ... + Builder builder = ImmutableSchemata.createBuilder(context); + builder.addTable("__ALLNODES__", "maker", "model", "year"); + schemata = builder.build(); + schemata = ImmutableSchemata.createBuilder(context) + .addTable("__ALLNODES__", "maker", "model", "year", "msrp") + .makeSearchable("__ALLNODES__", "maker") + .build(); + + // And create the SQL parser ... + sql = new SqlQueryParser(); } + protected Name name( String name ) { + return context.getValueFactories().getNameFactory().create(name); + } + protected Path path( String path ) { return context.getValueFactories().getPathFactory().create(path); } @@ -167,6 +196,10 @@ engine.index(workspaceName1, path("/Cars"), 10); } + // ---------------------------------------------------------------------------------------------------------------- + // Full-text search + // ---------------------------------------------------------------------------------------------------------------- + @Test public void shouldFindNodesByFullTextSearch() { engine.index(workspaceName1, path("/"), 100); @@ -190,4 +223,188 @@ assertThat(results.size(), is(1)); assertThat(results.get(0).getPath(), is(path("/Cars/Hybrid/Toyota Highlander"))); } + + @Test + public void shouldFindNodesBySimpleXpathQuery() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(18)); + System.out.println(results); + } + + // ---------------------------------------------------------------------------------------------------------------- + // Query + // ---------------------------------------------------------------------------------------------------------------- + + @Test + public void shouldFindNodesBySimpleQuery() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(18)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithEqualityComparisonCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE maker = 'Toyota'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Ignore + @Test + public void shouldFindNodesBySimpleQueryWithGreaterThanComparisonCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE mpgHighway > 20", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithLowercaseEqualityComparisonCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE LOWER(maker) = 'toyota'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithUppercaseEqualityComparisonCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE UPPER(maker) = 'TOYOTA'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithLikeComparisonCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE maker LIKE 'Toyo%'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithLikeComparisonCriteriaWithLeadingWildcard() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE maker LIKE '%yota'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithLowercaseLikeComparisonCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE LOWER(maker) LIKE 'toyo%'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(2)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithFullTextSearchCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE CONTAINS(maker,'martin')", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(1)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithDepthCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE DEPTH() > 2", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(12)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithLocalNameCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE LOCALNAME() LIKE 'Toyota%' OR LOCALNAME() LIKE 'Land %'", + context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(4)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithNameCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE NAME() LIKE 'Toyota%[1]' OR NAME() LIKE 'Land %'", + context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(4)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithNameCriteriaThatMatchesNoNodes() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE NAME() LIKE 'Toyota%[2]'", context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(0)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithPathCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE PATH() LIKE '/Cars[%]/Hy%/Toyota%' OR PATH() LIKE '/Cars[1]/Utility[1]/%'", + context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(6)); + System.out.println(results); + } + + @Test + public void shouldFindNodesBySimpleQueryWithDescendantCriteria() { + engine.index(workspaceName1, path("/"), 100); + QueryCommand query = sql.parseQuery("SELECT model, maker FROM __ALLNODES__ WHERE ISDESCENDANTNODE('/Cars/Hybrid')", + context); + QueryResults results = engine.query(context, workspaceName1, query, schemata); + assertNoErrors(results); + assertThat(results, is(notNullValue())); + assertThat(results.getRowCount(), is(3)); + System.out.println(results); + } + + protected void assertNoErrors( QueryResults results ) { + if (results.getProblems().hasErrors()) { + fail("Found errors: " + results.getProblems()); + } + assertThat(results.getProblems().hasErrors(), is(false)); + } + }