Author: rhauch Date: 2009-09-16 10:40:56 -0400 (Wed, 16 Sep 2009) New Revision: 1226 Added: trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndTokenizerTest.java Removed: trunk/dna-cnd/src/main/antlr/org/jboss/dna/cnd/Cnd.g trunk/dna-cnd/src/main/antlr3/org/jboss/dna/cnd/Cnd.g Modified: trunk/dna-cnd/.classpath trunk/dna-cnd/pom.xml trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndI18n.java trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndImporter.java trunk/dna-cnd/src/main/resources/org/jboss/dna/cnd/CndI18n.properties trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndImporterTest.java trunk/dna-cnd/src/test/resources/cnd/jcr-builtins-283-early-draft.cnd trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java trunk/dna-graph/src/main/java/org/jboss/dna/graph/JcrLexicon.java Log: DNA-522 CND files that use keywords as names cannot be read in ANTLR 3.0 by default treats keywords used in the parser as special tokens in the lexer, meaning that the lexer will always generate a keyword token whenever such a word appears in the input (regardless of whether that word is indeed a proper use of that keyword or just a coincidental match). To fix this requires a lot of nasty workarounds, and this makes the generated parser really large. So instead of using ANTLR, the parser was changed to use the TokenStream class added late last release. This is a very simple framework that makes it possible to write a parser that is fairly efficient but extremely easy to understand when reading and extremely easy to debug. Plus, the total lines of code for this CND importer/parser are actually reduced, even when excluding the generated code from the stats. The same CndImporter interface was kept, so there were no other changes outside of the 'dna-cnd' project. Also, one of the test cases used to this point attempted to import a namespace mapping where the URI was not quoted. Technically this is not possible according to the JCR 2.0 Public Final Draft specification, and the reference implementation does not appear to support it. Actually, there are several errors in the JCR 2.0 PFD specification section that talks about the CND grammar. In particular, the definition of a string is unclear in the very least and like in error, since it attempts to define a string as a sequence of one or more XmlChar, where XmlChar appears to be defined as any of the characters allowed in the Char production of the XML specification (http://www.w3.org/TR/xml/#NT-Char) and therefore would allow nearly any unicode character (including whitespace, newlines, etc). This is clearly NOT the behavior of the reference implementation, which treats a CND unquoted string simply as '[A-Za-z0-9:_]+'. Therefore, the string behavior of the parser was cleaned up ! a bit, though it still is more lenient than the reference implementation. Basically, our unquoted string is any non-whitespace character except the following: []<>=-+(),"'/{*| As a result of these changes, all existing unit tests and integration tests pass (though some of the dna-cnd tests were changed and new ones were added). Modified: trunk/dna-cnd/.classpath =================================================================== --- trunk/dna-cnd/.classpath 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/.classpath 2009-09-16 14:40:56 UTC (rev 1226) @@ -2,10 +2,8 @@ <classpath> <classpathentry kind="src" path="src/main/java"/> <classpathentry kind="src" path="src/main/resources"/> - <classpathentry kind="src" path="src/main/antlr3"/> <classpathentry kind="src" output="target/test-classes" path="src/test/java"/> <classpathentry kind="src" output="target/test-classes" path="src/test/resources"/> - <classpathentry kind="src" path="target/generated-sources/antlr3"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/> <classpathentry kind="output" path="target/classes"/> Modified: trunk/dna-cnd/pom.xml =================================================================== --- trunk/dna-cnd/pom.xml 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/pom.xml 2009-09-16 14:40:56 UTC (rev 1226) @@ -18,17 +18,6 @@ <groupId>org.jboss.dna</groupId> <artifactId>dna-graph</artifactId> </dependency> - <dependency> - <groupId>org.antlr</groupId> - <artifactId>antlr-runtime</artifactId> - <version>3.1.3</version> - <exclusions> - <exclusion> - <groupId>org.antlr</groupId> - <artifactId>stringtemplate</artifactId> - </exclusion> - </exclusions> - </dependency> <!-- Testing --> <dependency> @@ -42,24 +31,7 @@ <groupId>junit</groupId> <artifactId>junit</artifactId> </dependency> - <dependency> - <groupId>org.antlr</groupId> - <artifactId>antlr</artifactId> - <version>3.1.3</version> - <exclusions> - <exclusion> - <groupId>org.antlr</groupId> - <artifactId>stringtemplate</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.maven</groupId> - <artifactId>maven-project</artifactId> - </exclusion> - </exclusions> - <!-- we try to use scope to hide it from transitivity --> - <scope>test</scope> <!-- or perhaps 'provided' (see later discussion) or 'import' (maven >= 2.0.9) --> - </dependency> - <dependency> + <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-library</artifactId> </dependency> @@ -88,20 +60,4 @@ <artifactId>jcip-annotations</artifactId> </dependency> </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.antlr</groupId> - <artifactId>antlr3-maven-plugin</artifactId> - <version>3.1.3-1</version> - <executions> - <execution> - <goals> - <goal>antlr</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> </project> \ No newline at end of file Deleted: trunk/dna-cnd/src/main/antlr/org/jboss/dna/cnd/Cnd.g =================================================================== --- trunk/dna-cnd/src/main/antlr/org/jboss/dna/cnd/Cnd.g 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/main/antlr/org/jboss/dna/cnd/Cnd.g 2009-09-16 14:40:56 UTC (rev 1226) @@ -1,360 +0,0 @@ -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -grammar Cnd; - -options { - output=AST; -} - - -tokens { - NAMESPACES; - PREFIX; - URI; - NODE; - NAME; - PRIMARY_TYPE; - SUPERTYPES; - NODE_TYPES; - NODE_TYPE_ATTRIBUTES; - HAS_ORDERABLE_CHILD_NODES; - IS_MIXIN; - IS_ABSTRACT; - IS_QUERYABLE; - PRIMARY_ITEM_NAME; - PROPERTY_DEFINITION; - REQUIRED_TYPE; - DEFAULT_VALUES; - VALUE_CONSTRAINTS; - AUTO_CREATED; - MANDATORY; - PROTECTED; - REQUIRED_TYPE; - ON_PARENT_VERSION; - MULTIPLE; - QUERY_OPERATORS; - IS_FULL_TEXT_SEARCHABLE; - IS_QUERY_ORDERERABLE; - CHILD_NODE_DEFINITION; - REQUIRED_PRIMARY_TYPES; - DEFAULT_PRIMARY_TYPE; - SAME_NAME_SIBLINGS; -} - -@header { -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -package org.jboss.dna.cnd; -} - -@members { -@Override -public void emitErrorMessage( String msg ) { - // don't write messages to System.err ... - //super.emitErrorMessage(msg); -} -} - -@lexer::header { -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -package org.jboss.dna.cnd; -} - -@rulecatch { - catch (RecognitionException re) { - reportError(re); - } -} - -// ------------ -// Parser rules -// ------------ - -// These rules generate an AST that was designed with a structure that may be -// automatically processed into a graph structure. This structure is: -// -// NAMESPACES -// +- NODE (multiple) -// +- PREFIX -// +- string value -// +- URI -// +- string value -// NODE_TYPES -// +- NODE (multiple) -// +- NAME [nt:nodeType/@jcr:name] -// +- string value -// +- PRIMARY_TYPE [nt:base/@jcr:primaryType] -// +- string with value 'nt:nodeType' -// +- SUPERTYPES [nt:nodeType/@jcr:supertypes] -// +- string value(s) -// +- IS_ABSTRACT [nt:nodeType/@jcr:isAbstract] -// +- string containing boolean value (or false if not present) -// +- HAS_ORDERABLE_CHILD_NODES [nt:nodeType/@jcr:hasOrderableChildNodes] -// +- string containing boolean value (or false if not present) -// +- IS_MIXIN [nt:nodeType/@jcr:isMixin] -// +- string containing boolean value (or false if not present) -// +- IS_QUERYABLE [nt:nodeType/@jcr:isQueryable] -// +- string containing boolean value (or true if not present) -// +- PRIMARY_ITEM_NAME [nt:nodeType/@jcr:primaryItemName] -// +- string containing string value -// +- PROPERTY_DEFINITION [nt:nodeType/@jcr:propertyDefinition] -// +- NODE (multiple) -// +- NAME [nt:propertyDefinition/@jcr:name] -// +- string value -// +- PRIMARY_TYPE [nt:base/@jcr:primaryType] -// +- string with value 'nt:propertyDefinition' -// +- REQUIRED_TYPE [nt:propertyDefinition/@jcr:propertyType] -// +- string value (limited to one of the predefined types) -// +- DEFAULT_VALUES [nt:propertyDefinition/@jcr:defaultValues] -// +- string value(s) -// +- MULTIPLE [nt:propertyDefinition/@jcr:multiple] -// +- string containing boolean value (or false if not present) -// +- MANDATORY [nt:propertyDefinition/@jcr:mandatory] -// +- string containing boolean value (or false if not present) -// +- AUTO_CREATED [nt:propertyDefinition/@jcr:autoCreated] -// +- string containing boolean value (or false if not present) -// +- PROTECTED [nt:propertyDefinition/@jcr:protected] -// +- string containing boolean value (or false if not present) -// +- ON_PARENT_VERSION [nt:propertyDefinition/@jcr:onParentVersion] -// +- string value (limited to one of the predefined literal values) -// +- QUERY_OPERATORS -// +- string value (containing a comma-separated list of operator literals) -// +- IS_FULL_TEXT_SEARCHABLE [nt:propertyDefinition/@jcr:isFullTextSearchable] -// +- string containing boolean value (or true if not present) -// +- IS_QUERY_ORDERABLE [nt:propertyDefinition/@jcr:isQueryOrderable] -// +- string containing boolean value (or true if not present) -// +- VALUE_CONSTRAINTS [nt:propertyDefinition/@jcr:valueConstraints] -// +- string value(s) -// +- CHILD_NODE_DEFINITION [nt:nodeType/@jcr:childNodeDefinition] -// +- NODE (multiple) -// +- NAME [nt:childNodeDefinition/@jcr:name] -// +- string value -// +- PRIMARY_TYPE [nt:base/@jcr:primaryType] -// +- string with value 'nt:childNodeDefinition' -// +- REQUIRED_PRIMARY_TYPES [nt:childNodeDefinition/@jcr:requiredPrimaryTypes] -// +- string values (limited to names) -// +- DEFAULT_PRIMARY_TYPE [nt:childNodeDefinition/@jcr:defaultPrimaryType] -// +- string value (limited to a name) -// +- MANDATORY [nt:childNodeDefinition/@jcr:mandatory] -// +- string containing boolean value (or false if not present) -// +- AUTO_CREATED [nt:childNodeDefinition/@jcr:autoCreated] -// +- string containing boolean value (or false if not present) -// +- PROTECTED [nt:childNodeDefinition/@jcr:protected] -// +- string containing boolean value (or false if not present) -// +- SAME_NAME_SIBLINGS [nt:childNodeDefinition/@jcr:sameNameSiblings] -// +- string containing boolean value (or false if not present) -// +- ON_PARENT_VERSION [nt:childNodeDefinition/@jcr:onParentVersion] -// +- string value (limited to one of the predefined literal values) -// -// Comments -// -------- -// The JSR-283 specification states that comments are allowed in CND files but are to be removed prior -// to parsing and processing. This grammar accomplishes this by sending the MULTI_LINE_COMMENT and -// SINGLE_LINE_COMMENT tokens to the HIDDEN channel (along with all whitespace tokens). -// -// Case sensitivity -// ---------------- -// ANTLR 3 has no way of generating a case-insensitive lexer/parser (since this is dependent upon locale). -// However, it's possible to do this as outlined at http://www.antlr.org/wiki/pages/viewpage.action?pageId=1782. -// Note that the approach that overrides the "LT" method does not change the content but merely changes the -// character returned by the method, which are the characters used to match the rules. -// -// And this must be done for this grammar, since the CND tokens are case-insensitive. Note that this -// grammar defines all tokens as lower-case, so the ANTLRFileStream subclass must perform a 'toLowerCase()' -// rather than a 'toUpperCase()'. -// -// Post Processing -// --------------- -// A number of the values in the AST must be post-processed to perform the requisite validation. -// -// The string values may or may not be wrapped in quotes (single or double). Any quotes will -// need to be removed in post-processing. -// -// Also, the QUERY_OPERATORS string value should contain a comma-separated list of operators -// wrapped by single quotes. As noted below, this is a side effect of how this grammar's lexer -// automatically produces a STRING token whenever single-quoted strings are found, so by the -// time the parser rules are run, the operators are encapsulated in a STRING token. -// (It's arguable whether this is a defect in the CND grammar, or a side effect of how this grammar -// uses ANTLR; other parser and lexer rules were considered, but most caused a massive increase in -// the size of the generated code and a massive decrease in performance.) In short, these -// operators must be validated in a post-processing step. -// -// Generating JCR content -// ---------------------- -// -// The resulting AST's structure was designed such that it directly corresponds to the "nt:nodeType", -// "nt:propertyDefinition", and "nt:childNodeDefinition" node types that are specified in JSR-283 -// (proposed draft), in the hopes that this approach could be used with a "standard" AST tree walker -// that could automatically generate the JCR content. -// -// First of all, the structure follows a repeating multi-level pattern, where the "NODE" appears -// in the AST when a new node should be created, with JCR properties -// identified by AST nodes under "NODE" that have a single child (e.g., the property's value), -// or with JCR child nodes identified by AST nodes under "NODE" that also contain an AST "NODE" -// (with properties below it). -// -// The AST node names were designed to be easily translated into property names. ANTLR uses -// all-caps for AST node names and '_' to separate words in those names. To produce the JCR name, -// simply convert the first word to lowercase, and convert to lowercase all but the first character -// of each remaining word, and finally remove all '_' characters. -// -// This grammar (mostly) uses the "jcr" prefix (the namespaces should probably be "dna"). -// This could be automated by defining a mapping between the AST node names and the property names -// (as well as defining a default namespace for any AST node name that is to be converted automatically). -// -cnd : (namespaceMapping|nodeTypeDefinition)* EOF - -> ^(NAMESPACES namespaceMapping*)? ^(NODE_TYPES nodeTypeDefinition*)? ; - -// Namespace mappings -namespaceMapping : '<' prefix '=' uri '>' -> ^(NODE prefix uri); -prefix : STRING -> ^(PREFIX STRING); -uri : STRING -> ^(URI STRING); - -// Node type definitions -nodeTypeDefinition : nodeTypeName supertypes? nodeTypeOptions? ( propertyDefinition | childNodeDefinition )* - -> ^(NODE nodeTypeName ^(PRIMARY_TYPE STRING["nt:nodeType"]) supertypes? nodeTypeOptions? ^(PROPERTY_DEFINITION propertyDefinition*) ^(CHILD_NODE_DEFINITION childNodeDefinition*)) ; -nodeTypeName : '[' STRING ']' -> ^(NAME STRING) ; -supertypes : '>' stringList -> ^(SUPERTYPES stringList); -nodeTypeOptions : nodeTypeOption+; -nodeTypeOption : orderable | mixin | isAbstract | noQuery | primaryItem ; -orderable : ('o'|'ord'|'orderable') -> ^(HAS_ORDERABLE_CHILD_NODES STRING["true"]); -mixin : ('m' | 'mix' | 'mixin') -> ^(IS_MIXIN STRING["true"]); -isAbstract : ('a'|'abs'|'abstract') -> ^(IS_ABSTRACT STRING["true"]); -noQuery : ('nq'|'noquery') -> ^(IS_QUERYABLE STRING["false"]); -primaryItem : ('primaryitem'|'!') STRING -> ^(PRIMARY_ITEM_NAME STRING); - -// Property definitions ... -propertyDefinition : propertyName propertyType? defaultValues? ( propertyAttributes | valueConstraints )* - -> ^(NODE propertyName ^(PRIMARY_TYPE STRING["nt:propertyDefinition"]) propertyType? defaultValues? propertyAttributes* valueConstraints*); -propertyName : '-' STRING -> ^(NAME STRING); -propertyType : '(' propertyTypeLiteral ')' -> ^(REQUIRED_TYPE propertyTypeLiteral); -propertyTypeLiteral : ('string'|'binary'|'long'|'double'|'boolean'|'date'|'name'|'path'|'reference'|'*'); -defaultValues : '=' stringList -> ^(DEFAULT_VALUES stringList); -propertyAttributes : ( (onParentVersioningLiteral)=>onParentVersioning | (autoCreated)=> autoCreated | (multiple)=>multiple | (mandatory)=>mandatory | (isProtected)=>isProtected | (queryOperators)=>queryOperators | (noFullText)=>noFullText | (noQueryOrder)=>noQueryOrder)+ ; -//propertyAttributes : ( (onParentVersioning)=> onParentVersioning | ('a')=>autoCreated | ('m')=>mandatory |('p')=>isProtected | ('mul'|'*')=>multiple | ('q')=>queryOperators | ('nof')=>noFullText | ('nqord'|'noq')=>noQueryOrder)+ ; -valueConstraints : '<' stringList -> ^(VALUE_CONSTRAINTS stringList); -autoCreated : ('a'|'aut'|'autocreated') -> ^(AUTO_CREATED STRING["true"]); -mandatory : ('m'|'man'|'mandatory') -> ^(MANDATORY STRING["true"]); -isProtected : ('p'|'pro'|'protected') -> ^(PROTECTED STRING["true"]); -onParentVersioning : onParentVersioningLiteral -> ^(ON_PARENT_VERSION onParentVersioningLiteral); -onParentVersioningLiteral : ('copy'|'version'|'initialize'|'compute'|'ignore'|'abort'); -multiple : ('*'|'mul'|'multiple') -> ^(MULTIPLE STRING["true"]); -noFullText : ('nof'|'nofulltext') -> ^(IS_FULL_TEXT_SEARCHABLE STRING["false"]); -noQueryOrder : ('nqord'|'noqueryorder') -> ^(IS_QUERY_ORDERERABLE STRING["false"]); -queryOperators : ('qop'|'queryops') STRING -> ^(QUERY_OPERATORS STRING); -// The grammar defines the query operators to be wrapped by single quotes, and therefore the lexer produces a single STRING token. -// Since we cannot break this token up, we simply store the operators as a STRING literal, and will have to process -// at a later step. - -// Child node definitions ... -childNodeDefinition : nodeName requiredTypes? defaultType? nodeAttributes? - -> ^(NODE nodeName ^(PRIMARY_TYPE STRING["nt:childNodeDefinition"]) requiredTypes? defaultType? nodeAttributes?); -nodeName : '+' STRING -> ^(NAME STRING); -requiredTypes :'(' stringList ')' -> ^(REQUIRED_PRIMARY_TYPES stringList); -defaultType : '=' STRING -> ^(DEFAULT_PRIMARY_TYPE STRING); -nodeAttributes : nodeAttribute+; -nodeAttribute : autoCreated | mandatory | isProtected | onParentVersioning | sns ; -sns : ('sns'|'*') -> ^(SAME_NAME_SIBLINGS STRING["true"]) ; - -// General rules -stringList : STRING (',' STRING )* -> STRING*; - -// ------------ -// Lexer rules -// ------------ - -// Comments are currently sent to a separate channel -MULTI_LINE_COMMENT : ('/*' (options {greedy=false;} : . )* '*/') { $channel=HIDDEN;}; -SINGLE_LINE_COMMENT : '//' ~('\n' | '\r')* { $channel=HIDDEN;}; - -// Quoted strings allow for strings containing characters that otherwise would be CND delimiters. -// Note that the single- and double-quote characters must be escaped within the string. -// Also note that escape sequences are allowed as well. -STRING : QUOTED_STRING | UNQUOTED_STRING; - -// Quoted strings may contain escaped characters. -fragment QUOTED_STRING - : '"' ( EscapeSequence | ~('\\'|'"'))* '"' - | '\'' ( EscapeSequence | ~('\\'|'\''))* '\'' - ; - -fragment EscapeSequence - : '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\' - |('0'..'3') (('0'..'7') ('0'..'7')?)? - |'u'(('0'..'9')|('a'..'f')) (('0'..'9')|('a'..'f')) (('0'..'9')|('a'..'f')) - ) -; - -// An unquoted string is a word delimited by whitespace and CND tokens. -fragment UNQUOTED_STRING -: (~(' '|'\r'|'\t'|'\u000C'|'\n' // whitespace - |'='|'<'|'>'|'['|']'|','|'-'|'('|')' // tokens - ))+; - -WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} ; - - Deleted: trunk/dna-cnd/src/main/antlr3/org/jboss/dna/cnd/Cnd.g =================================================================== --- trunk/dna-cnd/src/main/antlr3/org/jboss/dna/cnd/Cnd.g 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/main/antlr3/org/jboss/dna/cnd/Cnd.g 2009-09-16 14:40:56 UTC (rev 1226) @@ -1,368 +0,0 @@ -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -grammar Cnd; - -options { - output=AST; -} - - -tokens { - NAMESPACES; - PREFIX; - URI; - NODE; - NAME; - PRIMARY_TYPE; - SUPERTYPES; - NODE_TYPES; - NODE_TYPE_ATTRIBUTES; - HAS_ORDERABLE_CHILD_NODES; - IS_MIXIN; - IS_ABSTRACT; - IS_QUERYABLE; - PRIMARY_ITEM_NAME; - PROPERTY_DEFINITION; - REQUIRED_TYPE; - DEFAULT_VALUES; - VALUE_CONSTRAINTS; - AUTO_CREATED; - MANDATORY; - PROTECTED; - REQUIRED_TYPE; - ON_PARENT_VERSION; - MULTIPLE; - IS_PRIMARY_PROPERTY; - QUERY_OPERATORS; - IS_FULL_TEXT_SEARCHABLE; - IS_QUERY_ORDERERABLE; - CHILD_NODE_DEFINITION; - REQUIRED_PRIMARY_TYPES; - DEFAULT_PRIMARY_TYPE; - SAME_NAME_SIBLINGS; -} - -@header { -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -package org.jboss.dna.cnd; -} - -@members { -@Override -public void emitErrorMessage( String msg ) { - // don't write messages to System.err ... - //super.emitErrorMessage(msg); -} -} - -@lexer::header { -/* - * JBoss DNA (http://www.jboss.org/dna) - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * See the AUTHORS.txt file in the distribution for a full listing of - * individual contributors. - * - * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA - * is licensed to you under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * JBoss DNA is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ -package org.jboss.dna.cnd; -} - -@rulecatch { - catch (RecognitionException re) { - reportError(re); - } -} - -// ------------ -// Parser rules -// ------------ - -// These rules generate an AST that was designed with a structure that may be -// automatically processed into a graph structure. This structure is: -// -// NAMESPACES -// +- NODE (multiple) -// +- PREFIX -// +- string value -// +- URI -// +- string value -// NODE_TYPES -// +- NODE (multiple) -// +- NAME [nt:nodeType/@jcr:name] -// +- string value -// +- PRIMARY_TYPE [nt:base/@jcr:primaryType] -// +- string with value 'nt:nodeType' -// +- SUPERTYPES [nt:nodeType/@jcr:supertypes] -// +- string value(s) -// +- IS_ABSTRACT [nt:nodeType/@jcr:isAbstract] -// +- string containing boolean value (or false if not present) -// +- HAS_ORDERABLE_CHILD_NODES [nt:nodeType/@jcr:hasOrderableChildNodes] -// +- string containing boolean value (or false if not present) -// +- IS_MIXIN [nt:nodeType/@jcr:isMixin] -// +- string containing boolean value (or false if not present) -// +- IS_QUERYABLE [nt:nodeType/@jcr:isQueryable] -// +- string containing boolean value (or true if not present) -// +- PRIMARY_ITEM_NAME [nt:nodeType/@jcr:primaryItemName] -// +- string containing string value -// +- PROPERTY_DEFINITION [nt:nodeType/@jcr:propertyDefinition] -// +- NODE (multiple) -// +- NAME [nt:propertyDefinition/@jcr:name] -// +- string value -// +- PRIMARY_TYPE [nt:base/@jcr:primaryType] -// +- string with value 'nt:propertyDefinition' -// +- REQUIRED_TYPE [nt:propertyDefinition/@jcr:propertyType] -// +- string value (limited to one of the predefined types) -// +- DEFAULT_VALUES [nt:propertyDefinition/@jcr:defaultValues] -// +- string value(s) -// +- MULTIPLE [nt:propertyDefinition/@jcr:multiple] -// +- string containing boolean value (or false if not present) -// +- MANDATORY [nt:propertyDefinition/@jcr:mandatory] -// +- string containing boolean value (or false if not present) -// +- AUTO_CREATED [nt:propertyDefinition/@jcr:autoCreated] -// +- string containing boolean value (or false if not present) -// +- PROTECTED [nt:propertyDefinition/@jcr:protected] -// +- string containing boolean value (or false if not present) -// +- ON_PARENT_VERSION [nt:propertyDefinition/@jcr:onParentVersion] -// +- string value (limited to one of the predefined literal values) -// +- QUERY_OPERATORS -// +- string value (containing a comma-separated list of operator literals) -// +- IS_FULL_TEXT_SEARCHABLE [nt:propertyDefinition/@jcr:isFullTextSearchable] -// +- string containing boolean value (or true if not present) -// +- IS_QUERY_ORDERABLE [nt:propertyDefinition/@jcr:isQueryOrderable] -// +- string containing boolean value (or true if not present) -// +- VALUE_CONSTRAINTS [nt:propertyDefinition/@jcr:valueConstraints] -// +- string value(s) -// +- CHILD_NODE_DEFINITION [nt:nodeType/@jcr:childNodeDefinition] -// +- NODE (multiple) -// +- NAME [nt:childNodeDefinition/@jcr:name] -// +- string value -// +- PRIMARY_TYPE [nt:base/@jcr:primaryType] -// +- string with value 'nt:childNodeDefinition' -// +- REQUIRED_PRIMARY_TYPES [nt:childNodeDefinition/@jcr:requiredPrimaryTypes] -// +- string values (limited to names) -// +- DEFAULT_PRIMARY_TYPE [nt:childNodeDefinition/@jcr:defaultPrimaryType] -// +- string value (limited to a name) -// +- MANDATORY [nt:childNodeDefinition/@jcr:mandatory] -// +- string containing boolean value (or false if not present) -// +- AUTO_CREATED [nt:childNodeDefinition/@jcr:autoCreated] -// +- string containing boolean value (or false if not present) -// +- PROTECTED [nt:childNodeDefinition/@jcr:protected] -// +- string containing boolean value (or false if not present) -// +- SAME_NAME_SIBLINGS [nt:childNodeDefinition/@jcr:sameNameSiblings] -// +- string containing boolean value (or false if not present) -// +- ON_PARENT_VERSION [nt:childNodeDefinition/@jcr:onParentVersion] -// +- string value (limited to one of the predefined literal values) -// -// JSR-283 node types -// ------------------ -// This grammar does support reading in the new 'decimal', 'weakreference' and 'uri' property types. Therefore, it -// is up to the calling component to verify that these types are allowed. Supporting these new types means -// that this CND grammar can be used for both JSR-170 and JSR-283. -// -// Comments -// -------- -// The JSR-283 specification states that comments are allowed in CND files but are to be removed prior -// to parsing and processing. This grammar accomplishes this by sending the MULTI_LINE_COMMENT and -// SINGLE_LINE_COMMENT tokens to the HIDDEN channel (along with all whitespace tokens). -// -// Case sensitivity -// ---------------- -// ANTLR 3 has no way of generating a case-insensitive lexer/parser (since this is dependent upon locale). -// However, it's possible to do this as outlined at http://www.antlr.org/wiki/pages/viewpage.action?pageId=1782. -// Note that the approach that overrides the "LT" method does not change the content but merely changes the -// character returned by the method, which are the characters used to match the rules. -// -// And this must be done for this grammar, since the CND tokens are case-insensitive. Note that this -// grammar defines all tokens as lower-case, so the ANTLRFileStream subclass must perform a 'toLowerCase()' -// rather than a 'toUpperCase()'. -// -// Post Processing -// --------------- -// A number of the values in the AST must be post-processed to perform the requisite validation. -// -// The string values may or may not be wrapped in quotes (single or double). Any quotes will -// need to be removed in post-processing. -// -// Also, the QUERY_OPERATORS string value should contain a comma-separated list of operators -// wrapped by single quotes. As noted below, this is a side effect of how this grammar's lexer -// automatically produces a STRING token whenever single-quoted strings are found, so by the -// time the parser rules are run, the operators are encapsulated in a STRING token. -// (It's arguable whether this is a defect in the CND grammar, or a side effect of how this grammar -// uses ANTLR; other parser and lexer rules were considered, but most caused a massive increase in -// the size of the generated code and a massive decrease in performance.) In short, these -// operators must be validated in a post-processing step. -// -// Generating JCR content -// ---------------------- -// -// The resulting AST's structure was designed such that it directly corresponds to the "nt:nodeType", -// "nt:propertyDefinition", and "nt:childNodeDefinition" node types that are specified in JSR-283 -// (proposed draft), in the hopes that this approach could be used with a "standard" AST tree walker -// that could automatically generate the JCR content. -// -// First of all, the structure follows a repeating multi-level pattern, where the "NODE" appears -// in the AST when a new node should be created, with JCR properties -// identified by AST nodes under "NODE" that have a single child (e.g., the property's value), -// or with JCR child nodes identified by AST nodes under "NODE" that also contain an AST "NODE" -// (with properties below it). -// -// The AST node names were designed to be easily translated into property names. ANTLR uses -// all-caps for AST node names and '_' to separate words in those names. To produce the JCR name, -// simply convert the first word to lowercase, and convert to lowercase all but the first character -// of each remaining word, and finally remove all '_' characters. -// -// This grammar (mostly) uses the "jcr" prefix (the namespaces should probably be "dna"). -// This could be automated by defining a mapping between the AST node names and the property names -// (as well as defining a default namespace for any AST node name that is to be converted automatically). -// -cnd : (namespaceMapping|nodeTypeDefinition)* EOF - -> ^(NAMESPACES namespaceMapping*)? ^(NODE_TYPES nodeTypeDefinition*)? ; - -// Namespace mappings -namespaceMapping : '<' prefix '=' uri '>' -> ^(NODE prefix uri); -prefix : - | 'mix' -> ^(PREFIX 'mix') // Must break out this built-in prefix because it happens to be a keyword - | STRING -> ^(PREFIX STRING); -uri : STRING -> ^(URI STRING); - -// Node type definitions -nodeTypeDefinition : nodeTypeName supertypes? nodeTypeOptions? ( propertyDefinition | childNodeDefinition )* - -> ^(NODE nodeTypeName ^(PRIMARY_TYPE STRING["nt:nodeType"]) supertypes? nodeTypeOptions? ^(PROPERTY_DEFINITION propertyDefinition*) ^(CHILD_NODE_DEFINITION childNodeDefinition*)) ; -nodeTypeName : '[' STRING ']' -> ^(NAME STRING) ; -supertypes : '>' stringList -> ^(SUPERTYPES stringList); -nodeTypeOptions : nodeTypeOption+; -nodeTypeOption : orderable | mixin | isAbstract | noQuery | primaryItem ; -orderable : ('o'|'ord'|'orderable') -> ^(HAS_ORDERABLE_CHILD_NODES STRING["true"]); -mixin : ('m' | 'mix' | 'mixin') -> ^(IS_MIXIN STRING["true"]); -isAbstract : ('a'|'abs'|'abstract') -> ^(IS_ABSTRACT STRING["true"]); -noQuery : ('nq'|'noquery') -> ^(IS_QUERYABLE STRING["false"]); -primaryItem : ('primaryitem'|'!') STRING -> ^(PRIMARY_ITEM_NAME STRING); - -// Property definitions ... -propertyDefinition : propertyName propertyType? defaultValues? ( propertyAttributes | valueConstraints )* - -> ^(NODE propertyName ^(PRIMARY_TYPE STRING["nt:propertyDefinition"]) propertyType? defaultValues? propertyAttributes* valueConstraints*); -propertyName : '-' ('*' -> ^(NAME '*') | STRING -> ^(NAME STRING)); -propertyType : '(' propertyTypeLiteral ')' -> ^(REQUIRED_TYPE propertyTypeLiteral); -propertyTypeLiteral : ('string'|'binary'|'long'|'double'|'boolean'|'decimal'|'date'|'name'|'path'|'reference'|'*'|'undefined'|'weakreference'|'uri'); -defaultValues : '=' stringList -> ^(DEFAULT_VALUES stringList); -propertyAttributes : ( (isPrimary)=>isPrimary | (onParentVersioningLiteral)=>onParentVersioning | (autoCreated)=> autoCreated | (multiple)=>multiple | (mandatory)=>mandatory | (isProtected)=>isProtected | (queryOperators)=>queryOperators | (noFullText)=>noFullText | (noQueryOrder)=>noQueryOrder)+ ; -//propertyAttributes : ( (onParentVersioning)=> onParentVersioning | ('a')=>autoCreated | ('m')=>mandatory |('p')=>isProtected | ('mul'|'*')=>multiple | ('q')=>queryOperators | ('nof')=>noFullText | ('nqord'|'noq')=>noQueryOrder)+ ; -valueConstraints : '<' stringList -> ^(VALUE_CONSTRAINTS stringList); -isPrimary : ('!'|'pri'|'primary') -> ^(IS_PRIMARY_PROPERTY STRING["true"]); -autoCreated : ('a'|'aut'|'autocreated') -> ^(AUTO_CREATED STRING["true"]); -mandatory : ('m'|'man'|'mandatory') -> ^(MANDATORY STRING["true"]); -isProtected : ('p'|'pro'|'protected') -> ^(PROTECTED STRING["true"]); -onParentVersioning : onParentVersioningLiteral -> ^(ON_PARENT_VERSION onParentVersioningLiteral); -onParentVersioningLiteral : ('copy'|'version'|'initialize'|'compute'|'ignore'|'abort'); -multiple : ('*'|'mul'|'multiple') -> ^(MULTIPLE STRING["true"]); -noFullText : ('nof'|'nofulltext') -> ^(IS_FULL_TEXT_SEARCHABLE STRING["false"]); -noQueryOrder : ('nqord'|'noqueryorder') -> ^(IS_QUERY_ORDERERABLE STRING["false"]); -queryOperators : ('qop'|'queryops') STRING -> ^(QUERY_OPERATORS STRING); -// The grammar defines the query operators to be wrapped by single quotes, and therefore the lexer produces a single STRING token. -// Since we cannot break this token up, we simply store the operators as a STRING literal, and will have to process -// at a later step. - -// Child node definitions ... -childNodeDefinition : nodeName requiredTypes? defaultType? nodeAttributes? - -> ^(NODE nodeName ^(PRIMARY_TYPE STRING["nt:childNodeDefinition"]) requiredTypes? defaultType? nodeAttributes?); -nodeName : '+' ('*' -> ^(NAME '*') | STRING -> ^(NAME STRING)); -requiredTypes :'(' stringList ')' -> ^(REQUIRED_PRIMARY_TYPES stringList); -defaultType : '=' STRING -> ^(DEFAULT_PRIMARY_TYPE STRING); -nodeAttributes : nodeAttribute+; -nodeAttribute : autoCreated | mandatory | (isPrimary)=>isPrimary| (isProtected)=>isProtected | onParentVersioning | sns ; -sns : ('sns'|'*'|'multiple') -> ^(SAME_NAME_SIBLINGS STRING["true"]) ; - -// General rules -stringList : STRING (',' STRING )* -> STRING*; - -// ------------ -// Lexer rules -// ------------ - -// Comments are currently sent to a separate channel -MULTI_LINE_COMMENT : ('/*' (options {greedy=false;} : . )* '*/') { $channel=HIDDEN;}; -SINGLE_LINE_COMMENT : '//' ~('\n' | '\r')* { $channel=HIDDEN;}; - -// Quoted strings allow for strings containing characters that otherwise would be CND delimiters. -// Note that the single- and double-quote characters must be escaped within the string. -// Also note that escape sequences are allowed as well. -STRING : QUOTED_STRING | UNQUOTED_STRING; - -// Quoted strings may contain escaped characters. -fragment QUOTED_STRING - : '"' ( EscapeSequence | ~('"'))* '"' - | '\'' ( EscapeSequence | ~('\''))* '\'' - ; - -fragment EscapeSequence - : '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\' - |('0'..'3') (('0'..'7') ('0'..'7')?)? - |'u'(('0'..'9')|('a'..'f')) (('0'..'9')|('a'..'f')) (('0'..'9')|('a'..'f')) - ) -; - -// An unquoted string is a word delimited by whitespace and CND tokens. -fragment UNQUOTED_STRING -: (~(' '|'\r'|'\t'|'\u000C'|'\n' // whitespace - |'='|'<'|'>'|'['|']'|','|'-'|'('|')' // tokens - ))+; - -WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} ; \ No newline at end of file Modified: trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndI18n.java =================================================================== --- trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndI18n.java 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndI18n.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -30,15 +30,11 @@ */ public final class CndI18n { - public static I18n passthrough; public static I18n errorImportingCndContent; - public static I18n expectedBooleanLiteral; - public static I18n expectedStringLiteral; public static I18n expectedValidNameLiteral; public static I18n expectedValidQueryOperator; - public static I18n expectedValidPropertyTypeName; - public static I18n expectedValidOnParentVersion; - public static I18n locationFromLineNumberAndCharacter; + public static I18n expectedNamespaceOrNodeDefinition; + public static I18n vendorBlockWasNotClosed; static { try { Modified: trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndImporter.java =================================================================== --- trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndImporter.java 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndImporter.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -23,7 +23,7 @@ */ package org.jboss.dna.cnd; -import java.io.ByteArrayInputStream; +import static org.jboss.dna.common.text.TokenStream.ANY_VALUE; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -34,30 +34,26 @@ import java.util.List; import java.util.Set; import net.jcip.annotations.NotThreadSafe; -import org.antlr.runtime.ANTLRFileStream; -import org.antlr.runtime.ANTLRInputStream; -import org.antlr.runtime.CharStream; -import org.antlr.runtime.CommonTokenStream; -import org.antlr.runtime.RecognitionException; -import org.antlr.runtime.TokenStream; -import org.antlr.runtime.tree.CommonTree; -import org.antlr.runtime.tree.RewriteCardinalityException; import org.jboss.dna.common.collection.Problems; -import org.jboss.dna.common.i18n.I18n; +import org.jboss.dna.common.text.TokenStream; +import org.jboss.dna.common.text.TokenStream.ParsingException; +import org.jboss.dna.common.text.TokenStream.Position; +import org.jboss.dna.common.text.TokenStream.Tokenizer; import org.jboss.dna.common.util.CheckArg; +import org.jboss.dna.common.util.IoUtil; import org.jboss.dna.graph.ExecutionContext; import org.jboss.dna.graph.JcrLexicon; import org.jboss.dna.graph.JcrNtLexicon; import org.jboss.dna.graph.io.Destination; import org.jboss.dna.graph.property.Name; import org.jboss.dna.graph.property.NameFactory; -import org.jboss.dna.graph.property.NamespaceRegistry; import org.jboss.dna.graph.property.Path; import org.jboss.dna.graph.property.PathFactory; +import org.jboss.dna.graph.property.Property; import org.jboss.dna.graph.property.PropertyFactory; -import org.jboss.dna.graph.property.ValueFactory; +import org.jboss.dna.graph.property.PropertyType; +import org.jboss.dna.graph.property.ValueFactories; import org.jboss.dna.graph.property.ValueFormatException; -import org.jboss.dna.graph.property.basic.LocalNamespaceRegistry; /** * A class that imports the node types contained in a JCR Compact Node Definition (CND) file into graph content. The content is @@ -71,70 +67,63 @@ @NotThreadSafe public class CndImporter { - private static final Set<String> VALID_PROPERTY_TYPES = Collections.unmodifiableSet(new HashSet<String>( - Arrays.asList(new String[] { - "STRING", - "BINARY", "LONG", - "DOUBLE", - "BOOLEAN", - "DECIMAL", - "DATE", "NAME", - "PATH", - "REFERENCE", - "WEAKREFERENCE", - "URI", - "UNDEFINED"}))); - private static final Set<String> VALID_ON_PARENT_VERSION = Collections.unmodifiableSet(new HashSet<String>( - Arrays.asList(new String[] { - "COPY", - "VERSION", - "INITIALIZE", - "COMPUTE", - "IGNORE", - "ABORT"}))); + protected final List<String> VALID_PROPERTY_TYPES = Collections.unmodifiableList(Arrays.asList(new String[] {"STRING", + "BINARY", "LONG", "DOUBLE", "BOOLEAN", "DATE", "NAME", "PATH", "REFERENCE", "WEAKREFERENCE", "DECIMAL", "URI", + "UNDEFINED", "*", "?"})); + + protected final List<String> VALID_ON_PARENT_VERSION = Collections.unmodifiableList(Arrays.asList(new String[] {"COPY", + "VERSION", "INITIALIZE", "COMPUTE", "IGNORE", "ABORT"})); + + protected final Set<String> VALID_QUERY_OPERATORS = Collections.unmodifiableSet(new HashSet<String>( + Arrays.asList(new String[] { + "=", "<>", "<", "<=", + ">", ">=", "LIKE"}))); + protected final Destination destination; - protected final Path parentPath; - private boolean debug = false; + protected final Path outputPath; + protected final PropertyFactory propertyFactory; + protected final PathFactory pathFactory; + protected final NameFactory nameFactory; + protected final ValueFactories valueFactories; + protected final boolean jcr170; /** * Create a new importer that will place the content in the supplied destination under the supplied path. * * @param destination the destination where content is to be written * @param parentPath the path in the destination below which the generated content is to appear + * @param compatibleWithPreJcr2 true if this parser should accept the CND format that was used in the reference implementation + * prior to JCR 2.0. * @throws IllegalArgumentException if either parameter is null */ public CndImporter( Destination destination, - Path parentPath ) { + Path parentPath, + boolean compatibleWithPreJcr2 ) { CheckArg.isNotNull(destination, "destination"); CheckArg.isNotNull(parentPath, "parentPath"); this.destination = destination; - this.parentPath = parentPath; + this.outputPath = parentPath; + ExecutionContext context = destination.getExecutionContext(); + this.valueFactories = context.getValueFactories(); + this.propertyFactory = context.getPropertyFactory(); + this.pathFactory = valueFactories.getPathFactory(); + this.nameFactory = valueFactories.getNameFactory(); + this.jcr170 = compatibleWithPreJcr2; } - void setDebug( boolean value ) { - this.debug = value; + /** + * Create a new importer that will place the content in the supplied destination under the supplied path. This parser will + * accept the CND format that was used in the reference implementation prior to JCR 2.0. + * + * @param destination the destination where content is to be written + * @param parentPath the path in the destination below which the generated content is to appear + * @throws IllegalArgumentException if either parameter is null + */ + public CndImporter( Destination destination, + Path parentPath ) { + this(destination, parentPath, true); } - protected ExecutionContext context() { - return this.destination.getExecutionContext(); - } - - protected NamespaceRegistry namespaces() { - return context().getNamespaceRegistry(); - } - - protected NameFactory nameFactory() { - return context().getValueFactories().getNameFactory(); - } - - protected ValueFactory<String> stringFactory() { - return context().getValueFactories().getStringFactory(); - } - - protected ValueFactory<Boolean> booleanFactory() { - return context().getValueFactories().getBooleanFactory(); - } - /** * Import the CND content from the supplied stream, placing the content into the importer's destination. * @@ -147,611 +136,572 @@ public void importFrom( InputStream stream, Problems problems, String resourceName ) throws IOException { - CndLexer lex = new CndLexer(new CaseInsensitiveInputStream(stream)); - importFrom(lex, resourceName, problems); + importFrom(IoUtil.read(stream), problems, resourceName); } /** * Import the CND content from the supplied stream, placing the content into the importer's destination. * - * @param content the string containing the CND content + * @param file the file containing the CND content * @param problems where any problems encountered during import should be reported - * @param resourceName a logical name for the resource name to be used when reporting problems; may be null if there is no - * useful name * @throws IOException if there is a problem reading from the supplied stream */ - public void importFrom( String content, - Problems problems, - String resourceName ) throws IOException { - ByteArrayInputStream stream = new ByteArrayInputStream(content.getBytes()); - importFrom(stream, problems, resourceName); + public void importFrom( File file, + Problems problems ) throws IOException { + importFrom(IoUtil.read(file), problems, file.getCanonicalPath()); } /** * Import the CND content from the supplied stream, placing the content into the importer's destination. * - * @param file the file containing the CND content + * @param content the string containing the CND content * @param problems where any problems encountered during import should be reported - * @throws IOException if there is a problem reading from the supplied stream + * @param resourceName a logical name for the resource name to be used when reporting problems; may be null if there is no + * useful name */ - public void importFrom( File file, - Problems problems ) throws IOException { - CndLexer lex = new CndLexer(new CaseInsensitiveFileStream(file.getAbsolutePath())); - importFrom(lex, file.getCanonicalPath(), problems); - } - - protected void importFrom( CndLexer lexer, - String resourceName, - Problems problems ) { - CommonTokenStream tokens = new CommonTokenStream(lexer); - CndParser parser = new Parser(tokens, problems, resourceName); - - // Create a new context with our own namespace registry ... - ImportContext context = new ImportContext(context(), problems, resourceName); - CommonTree ast = null; + public void importFrom( String content, + Problems problems, + String resourceName ) { try { - ast = (CommonTree)parser.cnd().getTree(); - } catch (RecognitionException e) { - // already handled by Parser, so we should not handle twice - } catch (RewriteCardinalityException e) { - // already handled by Parser, so we should not handle twice + parse(content); + destination.submit(); } catch (RuntimeException e) { problems.addError(e, CndI18n.errorImportingCndContent, (Object)resourceName, e.getMessage()); } + } - if (ast != null && problems.isEmpty()) { - - // -------------- - // Namespaces ... - // -------------- - - /* - NAMESPACES - +- NODE (multiple) - +- PREFIX - +- string value - +- URI - +- string value - */ - - // Get the namespaces before we do anything else ... - CommonTree namespaces = (CommonTree)ast.getFirstChildWithType(CndLexer.NAMESPACES); - if (namespaces != null) { - for (int i = 0; i != namespaces.getChildCount(); ++i) { - CommonTree namespace = (CommonTree)namespaces.getChild(i); - String prefix = namespace.getFirstChildWithType(CndLexer.PREFIX).getChild(0).getText(); - String uri = namespace.getFirstChildWithType(CndLexer.URI).getChild(0).getText(); - // Register the namespace ... - context.register(removeQuotes(prefix), removeQuotes(uri)); - } + /** + * Parse the CND content. + * + * @param content the content + * @throws ParsingException if there is a problem parsing the content + */ + protected void parse( String content ) { + Tokenizer tokenizer = new CndTokenizer(false, false); + TokenStream tokens = new TokenStream(content, tokenizer, false); + tokens.start(); + while (tokens.hasNext()) { + // Keep reading while we can recognize one of the two types of statements ... + if (tokens.matches("<", ANY_VALUE, "=", ANY_VALUE, ">")) { + parseNamespaceMapping(tokens); + } else if (tokens.matches("[", ANY_VALUE, "]")) { + parseNodeTypeDefinition(tokens, outputPath); + } else { + Position position = tokens.previousPosition(); + throw new ParsingException(position, CndI18n.expectedNamespaceOrNodeDefinition.text(tokens.consume(), + position.getLine(), + position.getColumn())); } + } + } - // -------------- - // Node Types ... - // -------------- + /** + * Parse the namespace mapping statement that is next on the token stream. + * + * @param tokens the tokens containing the namespace statement; never null + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseNamespaceMapping( TokenStream tokens ) { + tokens.consume('<'); + String prefix = removeQuotes(tokens.consume()); + tokens.consume('='); + String uri = removeQuotes(tokens.consume()); + tokens.consume('>'); + // Register the namespace ... + destination.getExecutionContext().getNamespaceRegistry().register(prefix, uri); + } - /* - NODE_TYPES - +- NODE (multiple) - +- NAME [nt:nodeType/@jcr:name] - +- string value - +- PRIMARY_TYPE [nt:base/@jcr:primaryType] - +- string with value 'nt:nodeType' - +- SUPERTYPES [nt:nodeType/@jcr:supertypes] - +- string value(s) - +- IS_ABSTRACT [nt:nodeType/@jcr:isAbstract] - +- string containing boolean value (or false if not present) - +- HAS_ORDERABLE_CHILD_NODES [nt:nodeType/@jcr:hasOrderableChildNodes] - +- string containing boolean value (or false if not present) - +- IS_MIXIN [nt:nodeType/@jcr:isMixin] - +- string containing boolean value (or false if not present) - +- IS_QUERYABLE [nt:nodeType/@jcr:isQueryable] - +- string containing boolean value (or true if not present) - +- PRIMARY_ITEM_NAME [nt:nodeType/@jcr:primaryItemName] - +- string containing string value - +- PROPERTY_DEFINITION [nt:nodeType/@jcr:propertyDefinition] - +- NODE (multiple) - +- NAME [nt:propertyDefinition/@jcr:name] - +- string value - +- PRIMARY_TYPE [nt:base/@jcr:primaryType] - +- string with value 'nt:propertyDefinition' - +- REQUIRED_TYPE [nt:propertyDefinition/@jcr:propertyType] - +- string value (limited to one of the predefined types) - +- DEFAULT_VALUES [nt:propertyDefinition/@jcr:defaultValues] - +- string value(s) - +- MULTIPLE [nt:propertyDefinition/@jcr:multiple] - +- string containing boolean value (or false if not present) - +- MANDATORY [nt:propertyDefinition/@jcr:mandatory] - +- string containing boolean value (or false if not present) - +- AUTO_CREATED [nt:propertyDefinition/@jcr:autoCreated] - +- string containing boolean value (or false if not present) - +- PROTECTED [nt:propertyDefinition/@jcr:protected] - +- string containing boolean value (or false if not present) - +- ON_PARENT_VERSION [nt:propertyDefinition/@jcr:onParentVersion] - +- string value (limited to one of the predefined literal values) - +- QUERY_OPERATORS - +- string value (containing a comma-separated list of operator literals) - +- IS_FULL_TEXT_SEARCHABLE [nt:propertyDefinition/@jcr:isFullTextSearchable] - +- string containing boolean value (or true if not present) - +- IS_QUERY_ORDERABLE [nt:propertyDefinition/@jcr:isQueryOrderable] - +- string containing boolean value (or true if not present) - +- VALUE_CONSTRAINTS [nt:propertyDefinition/@jcr:valueConstraints] - +- string value(s) - +- CHILD_NODE_DEFINITION [nt:nodeType/@jcr:childNodeDefinition] - +- NODE (multiple) - +- NAME [nt:childNodeDefinition/@jcr:name] - +- string value - +- PRIMARY_TYPE [nt:base/@jcr:primaryType] - +- string with value 'nt:childNodeDefinition' - +- REQUIRED_PRIMARY_TYPES [nt:childNodeDefinition/@jcr:requiredPrimaryTypes] - +- string values (limited to names) - +- DEFAULT_PRIMARY_TYPE [nt:childNodeDefinition/@jcr:defaultPrimaryType] - +- string value (limited to a name) - +- MANDATORY [nt:childNodeDefinition/@jcr:mandatory] - +- string containing boolean value (or false if not present) - +- AUTO_CREATED [nt:childNodeDefinition/@jcr:autoCreated] - +- string containing boolean value (or false if not present) - +- PROTECTED [nt:childNodeDefinition/@jcr:protected] - +- string containing boolean value (or false if not present) - +- SAME_NAME_SIBLINGS [nt:childNodeDefinition/@jcr:sameNameSiblings] - +- string containing boolean value (or false if not present) - +- ON_PARENT_VERSION [nt:childNodeDefinition/@jcr:onParentVersion] - +- string value (limited to one of the predefined literal values) - */ + /** + * Parse the node type definition that is next on the token stream. + * + * @param tokens the tokens containing the node type definition; never null + * @param path the path in the destination under which the node type definition should be stored; never null + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseNodeTypeDefinition( TokenStream tokens, + Path path ) { + // Parse the name, and create the path and a property for the name ... + Name name = parseNodeTypeName(tokens); + Path nodeTypePath = pathFactory.create(path, name); + List<Property> properties = new ArrayList<Property>(); + properties.add(propertyFactory.create(JcrLexicon.NODE_TYPE_NAME, name)); - // Get the node types ... - CommonTree nodeTypes = (CommonTree)ast.getFirstChildWithType(CndLexer.NODE_TYPES); - if (nodeTypes != null) { - int numNodeTypes = 0; - // Walk each of the nodes underneath the NODE_TYPES parent node ... - for (int i = 0; i != nodeTypes.getChildCount(); ++i) { - CommonTree nodeType = (CommonTree)nodeTypes.getChild(i); - if (this.debug) System.out.println(nodeType.toStringTree()); - Path nodeTypePath = context.createNodeType(nodeType, parentPath); - if (nodeTypePath == null) continue; - ++numNodeTypes; + // Read the (optional) supertypes ... + List<Name> supertypes = parseSupertypes(tokens); + properties.add(propertyFactory.create(JcrLexicon.SUPERTYPES, supertypes)); // even if empty - CommonTree propertyDefinitions = (CommonTree)nodeType.getFirstChildWithType(CndLexer.PROPERTY_DEFINITION); - if (propertyDefinitions != null) { - // Walk each of the nodes under PROPERTY_DEFINITION ... - for (int j = 0; j != propertyDefinitions.getChildCount(); ++j) { - CommonTree propDefn = (CommonTree)propertyDefinitions.getChild(j); - context.createPropertyDefinition(propDefn, nodeTypePath); - } - } + // Read the node type options ... + parseNodeTypeOptions(tokens, properties); + destination.create(nodeTypePath, properties); - CommonTree childNodeDefinitions = (CommonTree)nodeType.getFirstChildWithType(CndLexer.CHILD_NODE_DEFINITION); - if (childNodeDefinitions != null) { - // Walk each of the nodes under CHILD_NODE_DEFINITION ... - for (int j = 0; j != childNodeDefinitions.getChildCount(); ++j) { - CommonTree childDefn = (CommonTree)childNodeDefinitions.getChild(j); - context.createChildDefinition(childDefn, nodeTypePath); - } - } - } - - // Submit the destination - destination.submit(); - } - } + // Parse property and child node definitions ... + parsePropertyOrChildNodeDefinitions(tokens, nodeTypePath); } - protected final String removeQuotes( String text ) { - // Remove leading and trailing quotes, if there are any ... - return text.replaceFirst("^['\"]+", "").replaceAll("['\"]+$", ""); + /** + * Parse a node type name that appears next on the token stream. + * + * @param tokens the tokens containing the node type name; never null + * @return the node type name + * @throws ParsingException if there is a problem parsing the content + */ + protected Name parseNodeTypeName( TokenStream tokens ) { + tokens.consume('['); + Name name = parseName(tokens); + tokens.consume(']'); + return name; } /** - * Utility class that uses a context with a local namespace registry, along with the problems and resource name. + * Parse an optional list of supertypes if they appear next on the token stream. + * + * @param tokens the tokens containing the supertype names; never null + * @return the list of supertype names; never null, but possibly empty + * @throws ParsingException if there is a problem parsing the content */ - protected final class ImportContext { - private final ExecutionContext context; - private final ExecutionContext originalContext; - private final Problems problems; - private final String resourceName; + protected List<Name> parseSupertypes( TokenStream tokens ) { + if (tokens.canConsume('>')) { + // There is at least one supertype ... + return parseNameList(tokens); + } + return Collections.emptyList(); + } - protected ImportContext( ExecutionContext context, - Problems problems, - String resourceName ) { - // Create a context that has a local namespace registry - NamespaceRegistry localNamespaces = new LocalNamespaceRegistry(context.getNamespaceRegistry()); - this.originalContext = context; - this.context = context.with(localNamespaces); - this.problems = problems; - this.resourceName = resourceName; + /** + * Parse a list of strings, separated by commas. Any quotes surrounding the strings are removed. + * + * @param tokens the tokens containing the comma-separated strings; never null + * @return the list of string values; never null, but possibly empty + * @throws ParsingException if there is a problem parsing the content + */ + protected List<String> parseStringList( TokenStream tokens ) { + List<String> strings = new ArrayList<String>(); + if (tokens.canConsume('?')) { + // This list is variant ... + strings.add("?"); + } else { + // Read names until we see a ',' + do { + strings.add(removeQuotes(tokens.consume())); + } while (tokens.canConsume(',')); } + return strings; + } - protected ExecutionContext context() { - return this.context; + /** + * Parse a list of names, separated by commas. Any quotes surrounding the names are removed. + * + * @param tokens the tokens containing the comma-separated strings; never null + * @return the list of string values; never null, but possibly empty + * @throws ParsingException if there is a problem parsing the content + */ + protected List<Name> parseNameList( TokenStream tokens ) { + List<Name> names = new ArrayList<Name>(); + if (!tokens.canConsume('?')) { + // Read names until we see a ',' + do { + names.add(parseName(tokens)); + } while (tokens.canConsume(',')); } + return names; + } - protected void register( String prefix, - String uri ) { - // Register it in the local registry with the supplied prefix ... - context.getNamespaceRegistry().register(prefix, uri); - - // See if it is already registered in the original context ... - NamespaceRegistry registry = originalContext.getNamespaceRegistry(); - if (!registry.isRegisteredNamespaceUri(uri)) { - // It is not, so register it ... - registry.register(prefix, uri); + /** + * Parse the options for the node types, including whether the node type is orderable, a mixin, abstract, whether it supports + * querying, and which property/child node (if any) is the primary item for the node type. + * + * @param tokens the tokens containing the comma-separated strings; never null + * @param properties the list into which the properties that represent the options should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseNodeTypeOptions( TokenStream tokens, + List<Property> properties ) { + // Set up the defaults ... + boolean isOrderable = false; + boolean isMixin = false; + boolean isAbstract = false; + boolean isQueryable = true; + Name primaryItem = null; + String onParentVersion = "COPY"; + while (true) { + // Keep reading while we see a valid option ... + if (tokens.canConsumeAnyOf("ORDERABLE", "ORD", "O")) { + tokens.canConsume('?'); + isOrderable = true; + } else if (tokens.canConsumeAnyOf("MIXIN", "MIX", "M")) { + tokens.canConsume('?'); + isMixin = true; + } else if (tokens.canConsumeAnyOf("ABSTRACT", "ABS", "A")) { + tokens.canConsume('?'); + isAbstract = true; + } else if (tokens.canConsumeAnyOf("NOQUERY", "NOQ")) { + tokens.canConsume('?'); + isQueryable = false; + } else if (tokens.canConsumeAnyOf("PRIMARYITEM", "!")) { + primaryItem = parseName(tokens); + tokens.canConsume('?'); + } else if (tokens.matchesAnyOf(VALID_ON_PARENT_VERSION)) { + onParentVersion = tokens.consume(); + tokens.canConsume('?'); + } else if (tokens.matches("OPV")) { + // variant on-parent-version + onParentVersion = tokens.consume(); + tokens.canConsume('?'); + } else { + // No more valid options on the stream, so stop ... + break; } } - - protected NameFactory nameFactory() { - return this.context.getValueFactories().getNameFactory(); + properties.add(propertyFactory.create(JcrLexicon.HAS_ORDERABLE_CHILD_NODES, isOrderable)); + properties.add(propertyFactory.create(JcrLexicon.IS_MIXIN, isMixin)); + properties.add(propertyFactory.create(JcrLexicon.IS_ABSTRACT, isAbstract)); + properties.add(propertyFactory.create(JcrLexicon.IS_QUERYABLE, isQueryable)); + properties.add(propertyFactory.create(JcrLexicon.ON_PARENT_VERSION, onParentVersion.toUpperCase())); + if (primaryItem != null) { + properties.add(propertyFactory.create(JcrLexicon.PRIMARY_ITEM_NAME, primaryItem)); } + } - protected PathFactory pathFactory() { - return this.context.getValueFactories().getPathFactory(); + /** + * Parse a node type's property or child node definitions that appear next on the token stream. + * + * @param tokens the tokens containing the definitions; never null + * @param nodeTypePath the path in the destination where the node type has been created, and under which the property and + * child node type definitions should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parsePropertyOrChildNodeDefinitions( TokenStream tokens, + Path nodeTypePath ) { + while (true) { + // Keep reading while we see a property definition or child node definition ... + if (tokens.matches('-')) { + parsePropertyDefinition(tokens, nodeTypePath); + } else if (tokens.matches('+')) { + parseChildNodeDefinition(tokens, nodeTypePath); + } else { + // The next token does not signal either one of these, so stop ... + break; + } } + } - protected ValueFactory<String> stringFactory() { - return this.context.getValueFactories().getStringFactory(); - } + /** + * Parse a node type's property definition from the next tokens on the stream. + * + * @param tokens the tokens containing the definition; never null + * @param nodeTypePath the path in the destination where the node type has been created, and under which the property and + * child node type definitions should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parsePropertyDefinition( TokenStream tokens, + Path nodeTypePath ) { + tokens.consume('-'); + Name name = parseName(tokens); + Path path = pathFactory.create(nodeTypePath, JcrLexicon.PROPERTY_DEFINITION); + List<Property> properties = new ArrayList<Property>(); + properties.add(propertyFactory.create(JcrLexicon.NAME, name)); - protected ValueFactory<Boolean> booleanFactory() { - return this.context.getValueFactories().getBooleanFactory(); - } + // Parse the (optional) required type ... + parsePropertyType(tokens, properties, PropertyType.STRING.getName()); - protected void recordError( CommonTree node, - I18n msg, - Object... params ) { - String location = CndI18n.locationFromLineNumberAndCharacter.text(node.getLine(), node.getCharPositionInLine()); - problems.addError(msg, resourceName, location, params); - } + // Parse the default values ... + parseDefaultValues(tokens, properties); - protected void recordError( Throwable throwable, - CommonTree node, - I18n msg, - Object... params ) { - String location = CndI18n.locationFromLineNumberAndCharacter.text(node.getLine(), node.getCharPositionInLine()); - problems.addError(throwable, msg, resourceName, location, params); - } + // Parse the property attributes ... + parsePropertyAttributes(tokens, properties, name, path); - protected Name nameFrom( CommonTree node, - int childType ) { - CommonTree childNode = (CommonTree)node.getFirstChildWithType(childType); - if (childNode != null && childNode.getChildCount() > 0) { - CommonTree textNode = (CommonTree)childNode.getChild(0); - if (textNode.getToken().getTokenIndex() < 0) return null; - String text = removeQuotes(childNode.getChild(0).getText()); - try { - return nameFactory().create(text); - } catch (ValueFormatException e) { - recordError(e, node, CndI18n.expectedValidNameLiteral, text); - } - } - return null; - } + // Parse the property constraints ... + parseValueConstraints(tokens, properties); - protected Name[] namesFrom( CommonTree node, - int childType ) { - CommonTree childNode = (CommonTree)node.getFirstChildWithType(childType); - if (childNode != null && childNode.getChildCount() > 0) { - List<Name> names = new ArrayList<Name>(); - for (int i = 0; i != childNode.getChildCount(); ++i) { - String text = removeQuotes(childNode.getChild(i).getText()); - try { - names.add(nameFactory().create(text)); - } catch (ValueFormatException e) { - recordError(e, node, CndI18n.expectedValidNameLiteral, text); - } - } - return names.toArray(new Name[names.size()]); - } - return new Name[] {}; - } + // Create the node in the destination ... + destination.create(path, properties); + } - protected String stringFrom( CommonTree node, - int childType ) { - CommonTree childNode = (CommonTree)node.getFirstChildWithType(childType); - if (childNode != null && childNode.getChildCount() > 0) { - String text = removeQuotes(childNode.getChild(0).getText().trim()); - try { - return stringFactory().create(text); - } catch (ValueFormatException e) { - recordError(e, node, CndI18n.expectedStringLiteral, text); - } + /** + * Parse the property type, if a valid one appears next on the token stream. + * + * @param tokens the tokens containing the definition; never null + * @param properties the list into which the property that represents the property type should be placed + * @param defaultPropertyType the default property type if none is actually found + * @throws ParsingException if there is a problem parsing the content + */ + protected void parsePropertyType( TokenStream tokens, + List<Property> properties, + String defaultPropertyType ) { + if (tokens.canConsume('(')) { + // Parse the (optional) property type ... + String propertyType = defaultPropertyType; + if (tokens.matchesAnyOf(VALID_PROPERTY_TYPES)) { + propertyType = tokens.consume(); + if ("*".equals(propertyType)) propertyType = "UNDEFINED"; } - return null; + tokens.consume(')'); + properties.add(propertyFactory.create(JcrLexicon.REQUIRED_TYPE, propertyType.toUpperCase())); } + } - protected String[] stringsFrom( CommonTree node, - int childType ) { - CommonTree childNode = (CommonTree)node.getFirstChildWithType(childType); - if (childNode != null && childNode.getChildCount() > 0) { - List<String> names = new ArrayList<String>(); - for (int i = 0; i != childNode.getChildCount(); ++i) { - String text = removeQuotes(childNode.getChild(i).getText().trim()); - try { - names.add(stringFactory().create(text)); - } catch (ValueFormatException e) { - recordError(e, node, CndI18n.expectedStringLiteral, text); - } - } - return names.toArray(new String[names.size()]); + /** + * Parse the property definition's default value, if they appear next on the token stream. + * + * @param tokens the tokens containing the definition; never null + * @param properties the list into which the property that represents the default values should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseDefaultValues( TokenStream tokens, + List<Property> properties ) { + if (tokens.canConsume('=')) { + List<String> defaultValues = parseStringList(tokens); + if (!defaultValues.isEmpty()) { + properties.add(propertyFactory.create(JcrLexicon.DEFAULT_VALUES, defaultValues)); } - return new String[] {}; } + } - protected boolean booleanFrom( CommonTree node, - int childType, - boolean defaultValue ) { - CommonTree childNode = (CommonTree)node.getFirstChildWithType(childType); - if (childNode != null && childNode.getChildCount() > 0) { - String text = removeQuotes(childNode.getChild(0).getText()); - try { - return booleanFactory().create(text); - } catch (ValueFormatException e) { - recordError(e, node, CndI18n.expectedBooleanLiteral, text); - } + /** + * Parse the property definition's value constraints, if they appear next on the token stream. + * + * @param tokens the tokens containing the definition; never null + * @param properties the list into which the property that represents the value constraints should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseValueConstraints( TokenStream tokens, + List<Property> properties ) { + if (tokens.canConsume('<')) { + List<String> defaultValues = parseStringList(tokens); + if (!defaultValues.isEmpty()) { + properties.add(propertyFactory.create(JcrLexicon.VALUE_CONSTRAINTS, defaultValues)); } - return defaultValue; } + } - protected QueryOperator[] queryOperatorsFrom( CommonTree node, - int childType ) { - String text = stringFrom(node, childType); - if (text != null) { - String[] literals = text.split(","); - if (literals.length != 0) { - Set<QueryOperator> operators = new HashSet<QueryOperator>(); - for (String literal : literals) { - literal = literal.trim(); - if (literal.length() == 0) continue; - QueryOperator operator = QueryOperator.forText(literal); - if (operator != null) { - operators.add(operator); - } else { - recordError(node, CndI18n.expectedValidQueryOperator, literal); - } - } - return operators.toArray(new QueryOperator[operators.size()]); - } + /** + * Parse the property definition's attributes, if they appear next on the token stream. + * + * @param tokens the tokens containing the attributes; never null + * @param properties the list into which the properties that represents the attributes should be placed + * @param propDefnName the name of the property definition; never null + * @param propDefnPath the path in the destination to the property definition node; never null + * @throws ParsingException if there is a problem parsing the content + */ + protected void parsePropertyAttributes( TokenStream tokens, + List<Property> properties, + Name propDefnName, + Path propDefnPath ) { + boolean autoCreated = false; + boolean mandatory = false; + boolean isProtected = false; + boolean multiple = false; + boolean isFullTextSearchable = true; + boolean isQueryOrderable = true; + String onParentVersion = "COPY"; + while (true) { + if (tokens.canConsumeAnyOf("AUTOCREATED", "AUT", "A")) { + tokens.canConsume('?'); + autoCreated = true; + } else if (tokens.canConsumeAnyOf("MANDATORY", "MAN", "M")) { + tokens.canConsume('?'); + mandatory = true; + } else if (tokens.canConsumeAnyOf("PROTECTED", "PRO", "P")) { + tokens.canConsume('?'); + isProtected = true; + } else if (tokens.canConsumeAnyOf("MULTIPLE", "MUL", "*")) { + tokens.canConsume('?'); + multiple = true; + } else if (tokens.matchesAnyOf(VALID_ON_PARENT_VERSION)) { + onParentVersion = tokens.consume(); + tokens.canConsume('?'); + } else if (tokens.matches("OPV")) { + // variant on-parent-version + onParentVersion = tokens.consume(); + tokens.canConsume('?'); + } else if (tokens.canConsumeAnyOf("NOFULLTEXT", "NOF")) { + tokens.canConsume('?'); + isFullTextSearchable = false; + } else if (tokens.canConsumeAnyOf("NOQUERYORDER", "NQORD")) { + tokens.canConsume('?'); + isQueryOrderable = false; + } else if (tokens.canConsumeAnyOf("QUERYOPS", "QOP")) { + parseQueryOperators(tokens, properties); + } else if (jcr170 && tokens.canConsumeAnyOf("PRIMARY", "PRI", "!")) { + // Then this child node is considered the primary item ... + Property primaryItem = propertyFactory.create(JcrLexicon.PRIMARY_ITEM_NAME, propDefnName); + destination.setProperties(propDefnPath.getParent(), primaryItem); + } else { + break; } - return new QueryOperator[] {}; } + properties.add(propertyFactory.create(JcrLexicon.AUTO_CREATED, autoCreated)); + properties.add(propertyFactory.create(JcrLexicon.MANDATORY, mandatory)); + properties.add(propertyFactory.create(JcrLexicon.PROTECTED, isProtected)); + properties.add(propertyFactory.create(JcrLexicon.ON_PARENT_VERSION, onParentVersion.toUpperCase())); + properties.add(propertyFactory.create(JcrLexicon.MULTIPLE, multiple)); + properties.add(propertyFactory.create(JcrLexicon.IS_FULL_TEXT_SEARCHABLE, isFullTextSearchable)); + properties.add(propertyFactory.create(JcrLexicon.IS_QUERY_ORDERABLE, isQueryOrderable)); + } - protected String propertyTypeNameFrom( CommonTree node, - int childType ) { - String text = stringFrom(node, childType); - if ("*".equals(text)) text = "undefined"; - String upperText = text.toUpperCase(); - if (!VALID_PROPERTY_TYPES.contains(upperText)) { - recordError(node, CndI18n.expectedValidPropertyTypeName, text, VALID_PROPERTY_TYPES); - return null; - } - return upperText; + /** + * Parse the property definition's query operators, if they appear next on the token stream. + * + * @param tokens the tokens containing the definition; never null + * @param properties the list into which the property that represents the value constraints should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseQueryOperators( TokenStream tokens, + List<Property> properties ) { + if (tokens.canConsume('?')) { + return; } - - protected String onParentVersionFrom( CommonTree node, - int childType ) { - String text = stringFrom(node, childType); - if (text == null) return "COPY"; - String upperText = text.toUpperCase(); - if (!VALID_ON_PARENT_VERSION.contains(upperText)) { - recordError(node, CndI18n.expectedValidOnParentVersion, text, VALID_ON_PARENT_VERSION); - return null; + // The query operators are expected to be enclosed in a single quote, so therefore will be a single token ... + List<String> operators = new ArrayList<String>(); + String operatorList = removeQuotes(tokens.consume()); + // Now split this string on ',' ... + for (String operatorValue : operatorList.split(",")) { + String operator = operatorValue.trim(); + if (!VALID_QUERY_OPERATORS.contains(operator)) { + throw new ParsingException(tokens.previousPosition(), CndI18n.expectedValidQueryOperator.text(operator)); } - return upperText; + operators.add(operator); } - - protected Path createNodeType( CommonTree nodeType, - Path parentPath ) { - Name name = nameFrom(nodeType, CndLexer.NAME); - Name[] supertypes = namesFrom(nodeType, CndLexer.SUPERTYPES); - boolean isAbstract = booleanFrom(nodeType, CndLexer.IS_ABSTRACT, false); - boolean hasOrderableChildNodes = booleanFrom(nodeType, CndLexer.HAS_ORDERABLE_CHILD_NODES, false); - boolean isMixin = booleanFrom(nodeType, CndLexer.IS_MIXIN, false); - boolean isQueryable = booleanFrom(nodeType, CndLexer.IS_QUERYABLE, true); - Name primaryItemName = nameFrom(nodeType, CndLexer.PRIMARY_ITEM_NAME); - - if (primaryItemName == null) { - // See if one of the property definitions is marked as the primary ... - CommonTree propertyDefinitions = (CommonTree)nodeType.getFirstChildWithType(CndLexer.PROPERTY_DEFINITION); - if (propertyDefinitions != null) { - // Walk each of the nodes under PROPERTY_DEFINITION ... - for (int j = 0; j != propertyDefinitions.getChildCount(); ++j) { - CommonTree propDefn = (CommonTree)propertyDefinitions.getChild(j); - if (booleanFrom(propDefn, CndLexer.IS_PRIMARY_PROPERTY, false)) { - primaryItemName = nameFrom(propDefn, CndLexer.NAME); - break; - } - } - } - } - if (primaryItemName == null) { - // See if one of the child definitions is marked as the primary ... - CommonTree childNodeDefinitions = (CommonTree)nodeType.getFirstChildWithType(CndLexer.CHILD_NODE_DEFINITION); - if (childNodeDefinitions != null) { - // Walk each of the nodes under CHILD_NODE_DEFINITION ... - for (int j = 0; j != childNodeDefinitions.getChildCount(); ++j) { - CommonTree childDefn = (CommonTree)childNodeDefinitions.getChild(j); - if (booleanFrom(childDefn, CndLexer.IS_PRIMARY_PROPERTY, false)) { - primaryItemName = nameFrom(childDefn, CndLexer.NAME); - break; - } - } - } - } - - // Create the node for the node type ... - if (name == null) return null; - Path path = pathFactory().create(parentPath, name); - - PropertyFactory factory = context.getPropertyFactory(); - destination.create(path, - factory.create(JcrLexicon.PRIMARY_TYPE, JcrNtLexicon.NODE_TYPE), - factory.create(JcrLexicon.SUPERTYPES, (Object[])supertypes), - factory.create(JcrLexicon.IS_ABSTRACT, isAbstract), - factory.create(JcrLexicon.HAS_ORDERABLE_CHILD_NODES, hasOrderableChildNodes), - factory.create(JcrLexicon.IS_MIXIN, isMixin), - factory.create(JcrLexicon.IS_QUERYABLE, isQueryable), - factory.create(JcrLexicon.NODE_TYPE_NAME, name), - factory.create(JcrLexicon.PRIMARY_ITEM_NAME, primaryItemName)); - - return path; + if (operators.isEmpty()) { + operators.addAll(VALID_QUERY_OPERATORS); } + properties.add(propertyFactory.create(JcrLexicon.QUERY_OPERATORS, operators)); + } - protected Path createPropertyDefinition( CommonTree propDefn, - Path parentPath ) { - Name name = nameFrom(propDefn, CndLexer.NAME); - String requiredType = propertyTypeNameFrom(propDefn, CndLexer.REQUIRED_TYPE); - String[] defaultValues = stringsFrom(propDefn, CndLexer.DEFAULT_VALUES); - boolean multiple = booleanFrom(propDefn, CndLexer.MULTIPLE, false); - boolean mandatory = booleanFrom(propDefn, CndLexer.MANDATORY, false); - boolean autoCreated = booleanFrom(propDefn, CndLexer.AUTO_CREATED, false); - boolean isProtected = booleanFrom(propDefn, CndLexer.PROTECTED, false); - String onParentVersion = onParentVersionFrom(propDefn, CndLexer.ON_PARENT_VERSION); - /*QueryOperator[] queryOperators =*/queryOperatorsFrom(propDefn, CndLexer.QUERY_OPERATORS); - boolean isFullTextSearchable = booleanFrom(propDefn, CndLexer.IS_FULL_TEXT_SEARCHABLE, true); - boolean isQueryOrderable = booleanFrom(propDefn, CndLexer.IS_QUERY_ORDERERABLE, true); - String[] valueConstraints = stringsFrom(propDefn, CndLexer.VALUE_CONSTRAINTS); + /** + * Parse a node type's child node definition from the next tokens on the stream. + * + * @param tokens the tokens containing the definition; never null + * @param nodeTypePath the path in the destination where the node type has been created, and under which the child node type + * definitions should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseChildNodeDefinition( TokenStream tokens, + Path nodeTypePath ) { + tokens.consume('+'); + Name name = parseName(tokens); + Path path = pathFactory.create(nodeTypePath, JcrLexicon.CHILD_NODE_DEFINITION); + List<Property> properties = new ArrayList<Property>(); + properties.add(propertyFactory.create(JcrLexicon.NAME, name)); - // Create the node for the node type ... - if (name == null) return null; - Path path = pathFactory().create(parentPath, JcrLexicon.PROPERTY_DEFINITION); + parseRequiredPrimaryTypes(tokens, properties); + parseDefaultType(tokens, properties); + parseNodeAttributes(tokens, properties, name, path); - PropertyFactory factory = context.getPropertyFactory(); - destination.create(path, - factory.create(JcrLexicon.PRIMARY_TYPE, JcrNtLexicon.PROPERTY_DEFINITION), - factory.create(JcrLexicon.REQUIRED_TYPE, requiredType), - factory.create(JcrLexicon.DEFAULT_VALUES, (Object[])defaultValues), - factory.create(JcrLexicon.MULTIPLE, multiple), - factory.create(JcrLexicon.MANDATORY, mandatory), - factory.create(JcrLexicon.NAME, name), - factory.create(JcrLexicon.AUTO_CREATED, autoCreated), - factory.create(JcrLexicon.PROTECTED, isProtected), - factory.create(JcrLexicon.ON_PARENT_VERSION, onParentVersion), - // factory.create(DnaLexicon.QUERY_OPERATORS, queryOperators), - factory.create(JcrLexicon.IS_FULL_TEXT_SEARCHABLE, isFullTextSearchable), - factory.create(JcrLexicon.IS_QUERY_ORDERABLE, isQueryOrderable), - factory.create(JcrLexicon.VALUE_CONSTRAINTS, (Object[])valueConstraints)); - - return path; - } - - protected Path createChildDefinition( CommonTree childDefn, - Path parentPath ) { - Name name = nameFrom(childDefn, CndLexer.NAME); - Name[] requiredPrimaryTypes = namesFrom(childDefn, CndLexer.REQUIRED_PRIMARY_TYPES); - Name defaultPrimaryType = nameFrom(childDefn, CndLexer.DEFAULT_PRIMARY_TYPE); - boolean mandatory = booleanFrom(childDefn, CndLexer.MANDATORY, false); - boolean autoCreated = booleanFrom(childDefn, CndLexer.AUTO_CREATED, false); - boolean isProtected = booleanFrom(childDefn, CndLexer.PROTECTED, false); - String onParentVersion = onParentVersionFrom(childDefn, CndLexer.ON_PARENT_VERSION); - boolean sameNameSiblings = booleanFrom(childDefn, CndLexer.SAME_NAME_SIBLINGS, false); - - // Create the node for the node type ... - if (name == null) return null; - Path path = pathFactory().create(parentPath, JcrLexicon.CHILD_NODE_DEFINITION); - - PropertyFactory factory = context.getPropertyFactory(); - destination.create(path, - factory.create(JcrLexicon.PRIMARY_TYPE, JcrNtLexicon.CHILD_NODE_DEFINITION), - factory.create(JcrLexicon.REQUIRED_PRIMARY_TYPES, (Object[])requiredPrimaryTypes), - factory.create(JcrLexicon.DEFAULT_PRIMARY_TYPE, defaultPrimaryType), - factory.create(JcrLexicon.MANDATORY, mandatory), - factory.create(JcrLexicon.NAME, name), - factory.create(JcrLexicon.AUTO_CREATED, autoCreated), - factory.create(JcrLexicon.PROTECTED, isProtected), - factory.create(JcrLexicon.ON_PARENT_VERSION, onParentVersion), - factory.create(JcrLexicon.SAME_NAME_SIBLINGS, sameNameSiblings)); - - return path; - } + // Create the node in the destination ... + destination.create(path, properties); } - protected class Parser extends CndParser { - private final Problems problems; - private final String nameOfResource; - - public Parser( TokenStream input, - Problems problems, - String nameOfResource ) { - super(input); - this.problems = problems; - this.nameOfResource = nameOfResource; - } - - @Override - public void displayRecognitionError( String[] tokenNames, - RecognitionException e ) { - if (problems != null) { - String hdr = getErrorHeader(e); - String msg = getErrorMessage(e, tokenNames); - problems.addError(CndI18n.passthrough, nameOfResource, hdr, msg); - } else { - super.displayRecognitionError(tokenNames, e); + /** + * Parse the child node definition's list of required primary types, if they appear next on the token stream. + * + * @param tokens the tokens containing the definition; never null + * @param properties the list into which the property that represents the required types should be placed + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseRequiredPrimaryTypes( TokenStream tokens, + List<Property> properties ) { + if (tokens.canConsume('(')) { + List<Name> requiredTypes = parseNameList(tokens); + if (requiredTypes.isEmpty()) { + requiredTypes.add(JcrNtLexicon.BASE); } + properties.add(propertyFactory.create(JcrLexicon.REQUIRED_PRIMARY_TYPES, requiredTypes)); + tokens.consume(')'); } } /** - * Specialization of an {@link ANTLRInputStream} that converts all tokens to lowercase, allowing the grammar to be - * case-insensitive. See the <a href="http://www.antlr.org/wiki/pages/viewpage.action?pageId=1782&qu... documentation</a>. + * Parse the child node definition's default type, if they appear next on the token stream. + * + * @param tokens the tokens containing the definition; never null + * @param properties the list into which the property that represents the default primary type should be placed + * @throws ParsingException if there is a problem parsing the content */ - protected class CaseInsensitiveInputStream extends ANTLRInputStream { - protected CaseInsensitiveInputStream( InputStream stream ) throws IOException { - super(stream); + protected void parseDefaultType( TokenStream tokens, + List<Property> properties ) { + if (tokens.canConsume('=')) { + if (!tokens.canConsume('?')) { + Name defaultType = parseName(tokens); + properties.add(propertyFactory.create(JcrLexicon.DEFAULT_PRIMARY_TYPE, defaultType)); + } } + } - /** - * {@inheritDoc} - * - * @see org.antlr.runtime.ANTLRStringStream#LA(int) - */ - @Override - public int LA( int i ) { - if (i == 0) { - return 0; // undefined + /** + * Parse the child node definition's attributes, if they appear next on the token stream. + * + * @param tokens the tokens containing the attributes; never null + * @param properties the list into which the properties that represents the attributes should be placed + * @param childNodeDefnName the name of the child node definition; never null + * @param childNodeDefnPath the path in the destination to the child node definition node; never null + * @throws ParsingException if there is a problem parsing the content + */ + protected void parseNodeAttributes( TokenStream tokens, + List<Property> properties, + Name childNodeDefnName, + Path childNodeDefnPath ) { + boolean autoCreated = false; + boolean mandatory = false; + boolean isProtected = false; + boolean sns = false; + String onParentVersion = "COPY"; + while (true) { + if (tokens.canConsumeAnyOf("AUTOCREATED", "AUT", "A")) { + tokens.canConsume('?'); + autoCreated = true; + } else if (tokens.canConsumeAnyOf("MANDATORY", "MAN", "M")) { + tokens.canConsume('?'); + mandatory = true; + } else if (tokens.canConsumeAnyOf("PROTECTED", "PRO", "P")) { + tokens.canConsume('?'); + isProtected = true; + } else if (tokens.canConsumeAnyOf("SNS", "*")) { // standard JCR 2.0 keywords for SNS ... + tokens.canConsume('?'); + sns = true; + } else if (jcr170 && tokens.canConsumeAnyOf("MULTIPLE", "MUL", "*")) { // from pre-JCR 2.0 ref impl + tokens.canConsume('?'); + sns = true; + } else if (tokens.matchesAnyOf(VALID_ON_PARENT_VERSION)) { + onParentVersion = tokens.consume(); + tokens.canConsume('?'); + } else if (tokens.matches("OPV")) { + // variant on-parent-version + onParentVersion = tokens.consume(); + tokens.canConsume('?'); + } else if (jcr170 && tokens.canConsumeAnyOf("PRIMARY", "PRI", "!")) { + // Then this child node is considered the primary item ... + Property primaryItem = propertyFactory.create(JcrLexicon.PRIMARY_ITEM_NAME, childNodeDefnName); + destination.setProperties(childNodeDefnPath.getParent(), primaryItem); + } else { + break; } - if (i < 0) { - i++; // e.g., translate LA(-1) to use offset 0 - } - - if ((p + i - 1) >= n) { - return CharStream.EOF; - } - return Character.toLowerCase(data[p + i - 1]); } + properties.add(propertyFactory.create(JcrLexicon.AUTO_CREATED, autoCreated)); + properties.add(propertyFactory.create(JcrLexicon.MANDATORY, mandatory)); + properties.add(propertyFactory.create(JcrLexicon.PROTECTED, isProtected)); + properties.add(propertyFactory.create(JcrLexicon.ON_PARENT_VERSION, onParentVersion.toUpperCase())); + properties.add(propertyFactory.create(JcrLexicon.SAME_NAME_SIBLINGS, sns)); } /** - * Specialization of an {@link ANTLRInputStream} that converts all tokens to lowercase, allowing the grammar to be - * case-insensitive. See the <a href="http://www.antlr.org/wiki/pages/viewpage.action?pageId=1782&qu... documentation</a>. + * Parse the name that is expected to be next on the token stream. + * + * @param tokens the tokens containing the name; never null + * @return the name; never null + * @throws ParsingException if there is a problem parsing the content */ - protected class CaseInsensitiveFileStream extends ANTLRFileStream { - protected CaseInsensitiveFileStream( String fileName ) throws IOException { - super(fileName, null); + protected Name parseName( TokenStream tokens ) { + String value = tokens.consume(); + try { + return nameFactory.create(removeQuotes(value)); + } catch (ValueFormatException e) { + throw new ParsingException(tokens.previousPosition(), CndI18n.expectedValidNameLiteral.text(value)); } - - protected CaseInsensitiveFileStream( String fileName, - String encoding ) throws IOException { - super(fileName, encoding); - } - - /** - * {@inheritDoc} - * - * @see org.antlr.runtime.ANTLRStringStream#LA(int) - */ - @Override - public int LA( int i ) { - if (i == 0) { - return 0; // undefined - } - if (i < 0) { - i++; // e.g., translate LA(-1) to use offset 0 - } - - if ((p + i - 1) >= n) { - - return CharStream.EOF; - } - return Character.toLowerCase(data[p + i - 1]); - } } + protected final String removeQuotes( String text ) { + // Remove leading and trailing quotes, if there are any ... + return text.replaceFirst("^['\"]+", "").replaceAll("['\"]+$", ""); + } } Added: trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java =================================================================== --- trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java (rev 0) +++ trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -0,0 +1,224 @@ +package org.jboss.dna.cnd; + +import org.jboss.dna.common.CommonI18n; +import org.jboss.dna.common.text.TokenStream.CharacterStream; +import org.jboss.dna.common.text.TokenStream.ParsingException; +import org.jboss.dna.common.text.TokenStream.Position; +import org.jboss.dna.common.text.TokenStream.Tokenizer; +import org.jboss.dna.common.text.TokenStream.Tokens; + +/** + * A {@link Tokenizer} implementation that adheres to the CND format by ignoring whitespace while including tokens for individual + * symbols, the period ('.'), single-quoted strings, double-quoted strings, whitespace-delimited words, and optionally comments. + * This tokenizer optionally includes comments and vendor extensions. + */ +public class CndTokenizer implements Tokenizer { + /** + * The token type for tokens that represent an unquoted string containing a character sequence made up of non-whitespace and + * non-symbol characters. + */ + public static final int WORD = 1; + /** + * The token type for tokens that consist of an individual "symbol" character. The set of characters includes: + * <code>[]<>=-+(),</code> + */ + public static final int SYMBOL = 2; + /** + * The token type for tokens that consist of an individual '.' character. + */ + public static final int DECIMAL = 3; + /** + * The token type for tokens that consist of all the characters within single-quotes. Single quote characters are included if + * they are preceded (escaped) by a '\' character. + */ + public static final int SINGLE_QUOTED_STRING = 4; + /** + * The token type for tokens that consist of all the characters within double-quotes. Double quote characters are included if + * they are preceded (escaped) by a '\' character. + */ + public static final int DOUBLE_QUOTED_STRING = 5; + /** + * The token type for tokens that consist of all the characters between "/*" and "&#42;/" or between "//" and the next line + * terminator (e.g., '\n', '\r' or "\r\n"). + */ + public static final int COMMENT = 6; + /** + * The token type for the token containing a vendor extension block. + */ + public static final int VENDOR_EXTENSION = 7; + + private final boolean useComments; + private final boolean useVendorExtensions; + + public CndTokenizer( boolean useComments, + boolean useVendorExtensions ) { + this.useComments = useComments; + this.useVendorExtensions = useVendorExtensions; + } + + /** + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Tokenizer#tokenize(CharacterStream, Tokens) + */ + public void tokenize( CharacterStream input, + Tokens tokens ) throws ParsingException { + while (input.hasNext()) { + char c = input.next(); + switch (c) { + case ' ': + case '\t': + case '\n': + case '\r': + // Just skip these whitespace characters ... + break; + case '[': + case ']': + case '<': + case '>': + case '=': + case '-': + case '+': + case '(': + case ')': + case ',': + tokens.addToken(input.position(), input.index(), input.index() + 1, SYMBOL); + break; + // case '.': + // tokens.addToken(input.position(), input.index(), input.index() + 1, DECIMAL); + // break; + case '{': + // Vendor extension, meant to be excluded + int startIndex = input.index(); + Position startingPosition = input.position(); + boolean foundClosingBrace = false; + String vendorName = ""; + while (input.hasNext()) { + c = input.next(); + if (c == '\\' && input.isNext('}')) { + c = input.next(); // consume the '}' character since it is escaped + } else if (c == '}') { + foundClosingBrace = true; + break; + } + } + if (!foundClosingBrace) { + String msg = CndI18n.vendorBlockWasNotClosed.text(startingPosition.getLine(), + startingPosition.getColumn()); + throw new ParsingException(startingPosition, msg); + } + int endIndex = input.index() + 1; // beyond last character read + if (useVendorExtensions) { + tokens.addToken(startingPosition, startIndex, endIndex, VENDOR_EXTENSION); + } + break; + case '\"': + startIndex = input.index(); + startingPosition = input.position(); + boolean foundClosingQuote = false; + while (input.hasNext()) { + c = input.next(); + if (c == '\\' && input.isNext('"')) { + c = input.next(); // consume the ' character since it is escaped + } else if (c == '"') { + foundClosingQuote = true; + break; + } + } + if (!foundClosingQuote) { + String msg = CommonI18n.noMatchingDoubleQuoteFound.text(startingPosition.getLine(), + startingPosition.getColumn()); + throw new ParsingException(startingPosition, msg); + } + endIndex = input.index() + 1; // beyond last character read + tokens.addToken(startingPosition, startIndex, endIndex, DOUBLE_QUOTED_STRING); + break; + case '\'': + startIndex = input.index(); + startingPosition = input.position(); + foundClosingQuote = false; + while (input.hasNext()) { + c = input.next(); + if (c == '\\' && input.isNext('\'')) { + c = input.next(); // consume the ' character since it is escaped + } else if (c == '\'') { + foundClosingQuote = true; + break; + } + } + if (!foundClosingQuote) { + String msg = CommonI18n.noMatchingSingleQuoteFound.text(startingPosition.getLine(), + startingPosition.getColumn()); + throw new ParsingException(startingPosition, msg); + } + endIndex = input.index() + 1; // beyond last character read + tokens.addToken(startingPosition, startIndex, endIndex, SINGLE_QUOTED_STRING); + break; + case '/': + startIndex = input.index(); + startingPosition = input.position(); + if (input.isNext('/')) { + // End-of-line comment ... + boolean foundLineTerminator = false; + while (input.hasNext()) { + c = input.next(); + if (c == '\n' || c == '\r') { + foundLineTerminator = true; + break; + } + } + endIndex = input.index(); // the token won't include the '\n' or '\r' character(s) + if (!foundLineTerminator) ++endIndex; // must point beyond last char + if (c == '\r' && input.isNext('\n')) input.next(); + if (useComments) { + tokens.addToken(startingPosition, startIndex, endIndex, COMMENT); + } + } else if (input.isNext('*')) { + // Multi-line comment ... + while (input.hasNext() && !input.isNext('*', '/')) { + c = input.next(); + } + if (input.hasNext()) input.next(); // consume the '*' + if (input.hasNext()) input.next(); // consume the '/' + if (useComments) { + endIndex = input.index() + 1; // the token will include the '/' and '*' characters + tokens.addToken(startingPosition, startIndex, endIndex, COMMENT); + } + } else { + continue; + } + break; + default: + // The JCR 2.0 Public Final Draft is very unclear about what exactly a string is defined to be, + // and in fact the reference implementation (all versions) basically just treat an unquoted string + // to be defined as + // - unquoted_string ::= [A-Za-z0-9:_]+ + // But this doesn't really seem to align very well with the spec, which alludes to any number + // of XmlChar: + // - unquoted_string ::= XmlChar { XmlChar } + // - XmlChar ::= /* see §3.2.2 Local Names */ + // Then in Section 3.2.2, there is this rule: + // - XmlChar ::= /* Any character that matches the Char production at http://www.w3.org/TR/xml/#NT-Char */ + // This doesn't really make sense, because even whitespace is valid in Char. + // + // Could the CND grammar instead reference 3.2.5.2 (rather than 3.2.2)? This refers to qualified + // names, and appears to be much closer to the examples and reference implementation. + // + // What we're doing is basically reading all subsequent characters until we find a whitespace, + // one of the SYMBOL characters, a single- or double-quote character, a slash, or an open brace + // (since these are all the basis for other tokenization rules above). Also, the '*' and '|' + // characters terminate a WORD token, since these cannot appear unescaped within local names; + // since these do not appear in other rules above, they will result in one-character tokens. + // + startIndex = input.index(); + startingPosition = input.position(); + // Read as long as there is a valid XML character ... + while (input.hasNext() && !(input.isNextWhitespace() || input.isNextAnyOf("[]<>=-+(),\"'/{*|"))) { + c = input.next(); + } + endIndex = input.index() + 1; // beyond last character that was included + tokens.addToken(startingPosition, startIndex, endIndex, WORD); + } + } + } +} Property changes on: trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Modified: trunk/dna-cnd/src/main/resources/org/jboss/dna/cnd/CndI18n.properties =================================================================== --- trunk/dna-cnd/src/main/resources/org/jboss/dna/cnd/CndI18n.properties 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/main/resources/org/jboss/dna/cnd/CndI18n.properties 2009-09-16 14:40:56 UTC (rev 1226) @@ -22,12 +22,8 @@ # 02110-1301 USA, or see the FSF site: http://www.fsf.org. # -passthrough = {0} errorImportingCndContent = Error importing {0}: {1} -expectedBooleanLiteral = Expected 'true' or 'false', but was "{0}" -expectedStringLiteral = Expected a valid string, but was "{0}" expectedValidNameLiteral = Expected a valid name, but was "{0}" -expectedValidPropertyTypeName = Property type name was "{0}" but should be one of "{1}" -expectedValidOnParentVersion = On parent version value was "{0}" but should be one of "{1}" expectedValidQueryOperator = Expected a valid query operator ('=', '<>', '<', '<=', '>', '>=', or 'LIKE'), but was {0} -locationFromLineNumberAndCharacter = Line {0} column {1} \ No newline at end of file +expectedNamespaceOrNodeDefinition = Expected either a namespace mapping or a node type defininition, but found {0} at line {1}, column {2} +vendorBlockWasNotClosed = Vendor block at line {0}, column {1} was not closed Modified: trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndImporterTest.java =================================================================== --- trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndImporterTest.java 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndImporterTest.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -36,6 +36,7 @@ import java.util.Set; import org.jboss.dna.common.collection.Problem; import org.jboss.dna.common.collection.SimpleProblems; +import org.jboss.dna.common.text.TokenStream.ParsingException; import org.jboss.dna.graph.DnaLexicon; import org.jboss.dna.graph.ExecutionContext; import org.jboss.dna.graph.Graph; @@ -133,49 +134,85 @@ return result; } - @Test - public void shouldReportErrorIfTheNodeTypeNameIsEmpty() throws IOException { + @Test( expected = ParsingException.class ) + public void shouldReportErrorIfTheNodeTypeNameIsEmpty() { String cnd = "<ns = 'http://namespace.com/ns'> [] abstract"; - importer.importFrom(cnd, problems, "string"); - // printProblems(); - assertThat(problems.size(), is(1)); - assertThat(problems.iterator().next().getMessageString(), is("missing STRING at ']'")); + importer.parse(cnd); } - @Test - public void shouldReportErrorIfTheNodeTypeNameIsBlank() throws IOException { + @Test( expected = ParsingException.class ) + public void shouldReportErrorIfTheNodeTypeNameIsBlank() { String cnd = "<ns = 'http://namespace.com/ns'> [ ] abstract"; - importer.importFrom(cnd, problems, "string"); - // printProblems(); - assertThat(problems.size(), is(1)); - assertThat(problems.iterator().next().getMessageString(), is("missing STRING at ']'")); + importer.parse(cnd); } - @Test - public void shouldReportErrorIfTheNodeTypeNameIsNotFollowedByClosingBracket() throws IOException { + @Test( expected = ParsingException.class ) + public void shouldReportErrorIfTheNodeTypeNameIsNotFollowedByClosingBracket() { String cnd = "<ns = 'http://namespace.com/ns'> [ abstract"; - importer.importFrom(cnd, problems, "string"); - // printProblems(); - assertThat(problems.size(), is(1)); - assertThat(problems.iterator().next().getMessageString(), is("mismatched input 'abstract' expecting STRING")); + importer.parse(cnd); } - @Test - public void shouldReportErrorIfTheNodeTypeNameUsesInvalidNamespace() throws IOException { + @Test( expected = ParsingException.class ) + public void shouldReportErrorIfTheNodeTypeNameUsesInvalidNamespace() { String cnd = "<ns = 'http://namespace.com/ns'> [xyz:acme] abstract"; - importer.importFrom(cnd, problems, "string"); - // printProblems(); - assertThat(problems.size(), is(1)); - assertThat(problems.iterator().next().getMessage(), is(CndI18n.expectedValidNameLiteral)); + importer.parse(cnd); } @Test + public void shouldParseNamespaceDeclarationWithQuotedUriAndQuotedPrefix() { + String cnd = "<'ns' = 'http://namespace.com/ns'>"; + importer.parse(cnd); + } + + @Test + public void shouldParseNamespaceDeclarationWithUnquotedUriAndQuotedPrefix() { + String cnd = "<'ns' = http_namespace.com_ns>"; + importer.parse(cnd); + } + + @Test + public void shouldParseNamespaceDeclarationWithQuotedUriAndUnquotedPrefix() { + String cnd = "<ns = 'http://namespace.com/ns'>"; + importer.parse(cnd); + } + + @Test + public void shouldParseNamespaceDeclarationWithUnquotedUriAndUnquotedPrefix() { + String cnd = "<ns = http_namespace.com_ns>"; + importer.parse(cnd); + } + + @Test + public void shouldParseMinimalNodeDefinition() { + String cnd = "[nodeTypeName]"; + importer.parse(cnd); + } + + @Test + public void shouldParseMinimalNodeDefinitionWithSupertype() { + String cnd = "[nodeTypeName] > supertype"; + importer.parse(cnd); + } + + @Test + public void shouldParseMinimalNodeDefinitionWithSupertypes() { + String cnd = "[nodeTypeName] > supertype1, supertype2"; + importer.parse(cnd); + } + + @Test + public void shouldParseNodeDefinitionWithNameThatIsKeyword() { + String cnd = "[abstract] > supertype1, supertype2"; + importer.parse(cnd); + } + + @Test public void shouldImportCndThatUsesAllFeatures() throws IOException { // importer.setDebug(true); String cnd = "<ex = 'http://namespace.com/ns'>\n" + "[ex:NodeType] > ex:ParentType1, ex:ParentType2 abstract orderable mixin noquery primaryitem ex:property\n" - + "- ex:property (STRING) = 'default1', 'default2' mandatory autocreated protected multiple VERSION < 'constraint1', 'constraint2'\n" - + " queryops '=, <>, <, <=, >, >=, LIKE' nofulltext noqueryorder\n" + + "- ex:property (STRING) = 'default1', 'default2' mandatory autocreated protected multiple VERSION\n" + + " queryops '=, <>, <, <=, >, >=, LIKE' nofulltext noqueryorder < 'constraint1', 'constraint2'" + "+ ex:node (ex:reqType1, ex:reqType2) = ex:defaultType mandatory autocreated protected sns version"; importer.importFrom(cnd, problems, "string"); // assertThat(problems.size(), is(1)); @@ -503,7 +540,11 @@ assertThat(nodeType, hasProperty(JcrLexicon.IS_ABSTRACT, options.contains(NodeOptions.Abstract))); assertThat(nodeType, hasProperty(JcrLexicon.HAS_ORDERABLE_CHILD_NODES, options.contains(NodeOptions.Ordered))); assertThat(nodeType, hasProperty(JcrLexicon.IS_QUERYABLE, !options.contains(NodeOptions.Queryable))); - assertThat(nodeType, hasProperty(JcrLexicon.PRIMARY_ITEM_NAME, name(primaryItemName))); + if (primaryItemName != null) { + assertThat(nodeType, hasProperty(JcrLexicon.PRIMARY_ITEM_NAME, name(primaryItemName))); + } else { + assertThat(nodeType.getPropertiesByName().containsKey(JcrLexicon.PRIMARY_ITEM_NAME), is(false)); + } if (superTypes.length != 0) { Name[] superTypeNames = new Name[superTypes.length]; for (int i = 0; i != superTypes.length; ++i) { Added: trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndTokenizerTest.java =================================================================== --- trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndTokenizerTest.java (rev 0) +++ trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndTokenizerTest.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -0,0 +1,320 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.dna.cnd; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import java.util.LinkedList; +import org.jboss.dna.common.text.TokenStream.CharacterArrayStream; +import org.jboss.dna.common.text.TokenStream.ParsingException; +import org.jboss.dna.common.text.TokenStream.Position; +import org.jboss.dna.common.text.TokenStream.Tokens; +import org.junit.Before; +import org.junit.Test; + +public class CndTokenizerTest { + + private CndTokenizer tokenizer; + private Tokens tokenFactory; + private LinkedList<int[]> tokenValues; + + @Before + public void beforeEach() { + tokenizer = new CndTokenizer(true, false); + final LinkedList<int[]> tokenValues = new LinkedList<int[]>(); + tokenFactory = new Tokens() { + public void addToken( Position position, + int index ) { + int[] token = new int[] {index, index + 1, 0}; + tokenValues.add(token); + } + + public void addToken( Position position, + int startIndex, + int endIndex ) { + int[] token = new int[] {startIndex, endIndex, 0}; + tokenValues.add(token); + } + + public void addToken( Position position, + int startIndex, + int endIndex, + int type ) { + int[] token = new int[] {startIndex, endIndex, type}; + tokenValues.add(token); + } + }; + this.tokenValues = tokenValues; + } + + protected void tokenize( String input ) { + tokenizer.tokenize(new CharacterArrayStream(input.toCharArray()), tokenFactory); + } + + protected void assertNextTokenIs( int startIndex, + int endIndex, + int type ) { + int[] token = tokenValues.removeFirst(); + assertThat(token[0], is(startIndex)); + assertThat(token[1], is(endIndex)); + assertThat(token[2], is(type)); + } + + protected void assertNoMoreTokens() { + assertThat(tokenValues.isEmpty(), is(true)); + } + + @Test + public void shouldCreateNoTokensForEmptyContent() { + tokenize(""); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateNoTokensForContentWithOnlyWhitespace() { + tokenize(" \t \n \r\n \r "); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForEachSymbolCharacter() { + String content = "[]<>=-+(),"; + int numSymbols = content.length(); + tokenize(content); + for (int i = 0; i != numSymbols; ++i) { + assertNextTokenIs(i, i + 1, CndTokenizer.SYMBOL); + } + assertNoMoreTokens(); + } + + @Test + public void shouldNotIncludeColonInListOfSymbolsSinceTheyCanAppearInNames() { + tokenizer = new CndTokenizer(true, true); + String content = "dna:someName"; + int numSymbols = content.length(); + tokenize(content); + assertNextTokenIs(0, content.length(), CndTokenizer.WORD); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateVendorExtensionToken() { + tokenizer = new CndTokenizer(true, true); + String content = "{vendor extension}"; + int numSymbols = content.length(); + tokenize(content); + assertNextTokenIs(0, content.length(), CndTokenizer.VENDOR_EXTENSION); + assertNoMoreTokens(); + } + + @Test + public void shouldNotCreateVendorExtensionTokenIfTokenizerIsNotUsingThem() { + tokenizer = new CndTokenizer(true, false); + String content = "{vendor extension}"; + int numSymbols = content.length(); + tokenize(content); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForEndOfLineComment() { + String content = "--//this is a comment\n"; + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, content.length() - 1, CndTokenizer.COMMENT); // -1 because '\n' is not included + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForEndOfLineCommentThatEndsWithEndOfString() { + String content = "--//this is a comment"; + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForMultiLineComment() { + String content = "--/*this is a comment*/-"; + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, content.length() - 1, CndTokenizer.COMMENT); + assertNextTokenIs(content.length() - 1, content.length(), CndTokenizer.SYMBOL); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForMultiLineCommentAtEndOfContent() { + String content = "--/*this is a comment*/"; + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForMultiLineCommentWithoutTerminatingCharacters() { + String content = "--/*this is a comment"; + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForMultiLineCommentWithoutAllTerminatingCharacters() { + String content = "--/*this is a comment*"; + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForSingleQuotedString() { + String content = "--'this is a single-quoted \n string'-"; + assertThat(content.charAt(2), is('\'')); + assertThat(content.charAt(35), is('\'')); + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, 36, CndTokenizer.SINGLE_QUOTED_STRING); + assertNextTokenIs(36, 37, CndTokenizer.SYMBOL); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForSingleQuotedStringWithEscapedSingleQuoteCharacters() { + String content = "--'this \"is\" a \\'single-quoted\\' \n string'-"; + assertThat(content.charAt(2), is('\'')); + assertThat(content.charAt(41), is('\'')); + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, 42, CndTokenizer.SINGLE_QUOTED_STRING); + assertNextTokenIs(42, 43, CndTokenizer.SYMBOL); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForSingleQuotedStringAtEndOfContent() { + String content = "--'this is a single-quoted \n string'"; + assertThat(content.charAt(2), is('\'')); + assertThat(content.charAt(35), is('\'')); + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, 36, CndTokenizer.SINGLE_QUOTED_STRING); + assertNoMoreTokens(); + } + + @Test( expected = ParsingException.class ) + public void shouldCreateTokenForSingleQuotedStringWithoutClosingQuote() { + String content = "--'this is a single-quoted \n string"; + tokenize(content); + } + + @Test + public void shouldCreateTokenForDoubleQuotedString() { + String content = "--\"this is a double-quoted \n string\"-"; + assertThat(content.charAt(2), is('"')); + assertThat(content.charAt(35), is('"')); + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, 36, CndTokenizer.DOUBLE_QUOTED_STRING); + assertNextTokenIs(36, 37, CndTokenizer.SYMBOL); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForDoubleQuotedStringWithEscapedDoubleQuoteCharacters() { + String content = "--\"this 'is' a \\\"double-quoted\\\" \n string\"-"; + assertThat(content.charAt(2), is('"')); + assertThat(content.charAt(41), is('"')); + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, 42, CndTokenizer.DOUBLE_QUOTED_STRING); + assertNextTokenIs(42, 43, CndTokenizer.SYMBOL); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokenForDoubleQuotedStringAtEndOfContent() { + String content = "--\"this is a double-quoted \n string\""; + assertThat(content.charAt(2), is('"')); + assertThat(content.charAt(35), is('"')); + tokenize(content); + assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); + assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); + assertNextTokenIs(2, 36, CndTokenizer.DOUBLE_QUOTED_STRING); + assertNoMoreTokens(); + } + + @Test( expected = ParsingException.class ) + public void shouldCreateTokenForDoubleQuotedStringWithoutClosingQuote() { + String content = "--\"this is a double-quoted \n string"; + tokenize(content); + } + + @Test + public void shouldCreateTokensForWordsWithAlphabeticCharacters() { + String content = "This is a series of words."; + tokenize(content); + assertNextTokenIs(0, 4, CndTokenizer.WORD); + assertNextTokenIs(5, 7, CndTokenizer.WORD); + assertNextTokenIs(8, 9, CndTokenizer.WORD); + assertNextTokenIs(10, 16, CndTokenizer.WORD); + assertNextTokenIs(17, 19, CndTokenizer.WORD); + assertNextTokenIs(20, 26, CndTokenizer.WORD); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokensForWordsWithNumericCharacters() { + String content = "1234 4 5353.324"; + tokenize(content); + assertNextTokenIs(0, 4, CndTokenizer.WORD); + assertNextTokenIs(5, 6, CndTokenizer.WORD); + assertNextTokenIs(7, 15, CndTokenizer.WORD); + assertNoMoreTokens(); + } + + @Test + public void shouldCreateTokensForWordsWithAlphaNumericCharacters() { + String content = "123a 5353.324e100"; + tokenize(content); + assertNextTokenIs(0, 4, CndTokenizer.WORD); + assertNextTokenIs(5, 17, CndTokenizer.WORD); + assertNoMoreTokens(); + } +} Property changes on: trunk/dna-cnd/src/test/java/org/jboss/dna/cnd/CndTokenizerTest.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Modified: trunk/dna-cnd/src/test/resources/cnd/jcr-builtins-283-early-draft.cnd =================================================================== --- trunk/dna-cnd/src/test/resources/cnd/jcr-builtins-283-early-draft.cnd 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-cnd/src/test/resources/cnd/jcr-builtins-283-early-draft.cnd 2009-09-16 14:40:56 UTC (rev 1226) @@ -1,7 +1,7 @@ -<jcr=http://www.jcp.org/jcr/1.0> -<nt=http://www.jcp.org/jcr/nt/1.0> -<mix=http://www.jcp.org/jcr/mix/1.0> -<sv=http://www.jcp.org/jcr/sv/1.0> +<jcr='http://www.jcp.org/jcr/1.0'> +<nt='http://www.jcp.org/jcr/nt/1.0'> +<mix='http://www.jcp.org/jcr/mix/1.0'> +<sv='http://www.jcp.org/jcr/sv/1.0'> // ------------------------------------------------------------------------ // Pre-defined Node Types Modified: trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java =================================================================== --- trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -32,6 +32,7 @@ import net.jcip.annotations.NotThreadSafe; import org.jboss.dna.common.CommonI18n; import org.jboss.dna.common.util.CheckArg; +import org.jboss.dna.common.xml.XmlCharacters; /** * A foundation for basic parsers that tokenizes input content and allows parsers to easily access and use those tokens. A @@ -421,6 +422,28 @@ } /** + * Get the position of the previous token. + * + * @return the previous token's position; never null + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + * @throws NoSuchElementException if there is no previous token + */ + public Position previousPosition() { + return previousToken().position(); + } + + /** + * Get the position of the next (or current) token. + * + * @return the current token's position; never null + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + * @throws NoSuchElementException if there is no previous token + */ + public Position nextPosition() { + return currentToken().position(); + } + + /** * Return the value of this token and move to the next token. * * @return the value of the current token @@ -746,6 +769,87 @@ } /** + * Attempt to consume the next token if it matches one of the supplied values. + * + * @param firstOption the first option for the value of the current token + * @param additionalOptions the additional options for the value of the current token + * @return true if the current token's value did match one of the suplied options, or false otherwise + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + */ + public boolean canConsumeAnyOf( String firstOption, + String... additionalOptions ) throws IllegalStateException { + if (completed) return false; + if (canConsume(firstOption)) return true; + for (String nextOption : additionalOptions) { + if (canConsume(nextOption)) return true; + } + return false; + } + + /** + * Attempt to consume the next token if it matches one of the supplied values. + * + * @param options the options for the value of the current token + * @return true if the current token's value did match one of the suplied options, or false otherwise + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + */ + public boolean canConsumeAnyOf( String[] options ) throws IllegalStateException { + if (completed) return false; + for (String option : options) { + if (canConsume(option)) return true; + } + return false; + } + + /** + * Attempt to consume the next token if it matches one of the supplied values. + * + * @param options the options for the value of the current token + * @return true if the current token's value did match one of the suplied options, or false otherwise + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + */ + public boolean canConsumeAnyOf( Iterable<String> options ) throws IllegalStateException { + if (completed) return false; + for (String option : options) { + if (canConsume(option)) return true; + } + return false; + } + + /** + * Attempt to consume the next token if it matches one of the supplied types. + * + * @param firstTypeOption the first option for the type of the current token + * @param additionalTypeOptions the additional options for the type of the current token + * @return true if the current token's type matched one of the supplied options, or false otherwise + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + */ + public boolean canConsumeAnyOf( int firstTypeOption, + int... additionalTypeOptions ) throws IllegalStateException { + if (completed) return false; + if (canConsume(firstTypeOption)) return true; + for (int nextTypeOption : additionalTypeOptions) { + if (canConsume(nextTypeOption)) return true; + } + return false; + } + + /** + * Attempt to consume the next token if it matches one of the supplied types. + * + * @param typeOptions the options for the type of the current token + * @return true if the current token's type matched one of the supplied options, or false otherwise + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + */ + public boolean canConsumeAnyOf( int[] typeOptions ) throws IllegalStateException { + if (completed) return false; + for (int nextTypeOption : typeOptions) { + if (canConsume(nextTypeOption)) return true; + } + return false; + } + + /** * Determine if the current token matches the expected value. * <p> * The {@link #ANY_VALUE ANY_VALUE} constant can be used as a wildcard. @@ -1003,13 +1107,30 @@ return tokenIterator.hasNext(); } - // public Position currentPosition() { - // return currentToken().position(); - // } - // - // public int currentType() { - // return currentToken().type(); - // } + /** + * {@inheritDoc} + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + ListIterator<Token> iter = tokens.listIterator(tokenIterator.previousIndex()); + StringBuilder sb = new StringBuilder(); + if (iter.hasNext()) { + sb.append(iter.next()); + int count = 1; + while (iter.hasNext()) { + if (count > 20) { + sb.append(" ..."); + break; + } + sb.append(" "); + ++count; + sb.append(iter.next()); + } + } + return sb.toString(); + } private void moveToNextToken() { // And move the currentToken to the next token ... @@ -1039,6 +1160,30 @@ return currentToken; } + /** + * Get the previous token. This does not modify the state. + * + * @return the previous token; never null + * @throws IllegalStateException if this method was called before the stream was {@link #start() started} + * @throws NoSuchElementException if there is no previous token + */ + final Token previousToken() throws IllegalStateException, NoSuchElementException { + if (currentToken == null) { + if (completed) { + if (tokens.isEmpty()) { + throw new NoSuchElementException(CommonI18n.noMoreContent.text()); + } + return tokens.get(tokens.size() - 1); + } + throw new IllegalStateException(CommonI18n.startMethodMustBeCalledBeforeConsumingOrMatching.text()); + } + if (!tokenIterator.hasPrevious()) { + throw new NoSuchElementException(CommonI18n.noMoreContent.text()); + } + ListIterator<Token> temp = tokens.listIterator(tokenIterator.previousIndex()); + return temp.next(); + } + String generateFragment() { // Find the current position ... assert currentToken != null; @@ -1133,6 +1278,14 @@ boolean isNextWhitespace(); /** + * Determine if the next character on the sream is a {@link XmlCharacters#isValid(int) valid XML character}. This method + * does <i>not</i> advance the stream. + * + * @return true if there is a {@link #next() next} character and it is a valid XML character, or false otherwise + */ + boolean isNextValidXmlCharacter(); + + /** * Determine if the next character on the sream is the supplied value. This method does <i>not</i> advance the stream. * * @param c the character value to compare to the next character on the stream @@ -1234,26 +1387,74 @@ /** * The interface defining a token, which references the characters in the actual input character stream. + * + * @see CaseSensitiveTokenFactory + * @see CaseInsensitiveTokenFactory */ @Immutable - protected interface Token { + public interface Token { + /** + * Get the value of the token, in actual case. + * + * @return the value + */ String value(); + /** + * Determine if the token matches the supplied string. + * + * @param expected the expected value + * @return true if the token's value matches the supplied value, or false otherwise + */ boolean matches( String expected ); + /** + * Determine if the token matches the supplied character. + * + * @param expected the expected character value + * @return true if the token's value matches the supplied character value, or false otherwise + */ boolean matches( char expected ); + /** + * Get the type of the token. + * + * @return the token's type + */ int type(); + /** + * Get the index in the raw stream for the first character in the token. + * + * @return the starting index of the token + */ int startIndex(); + /** + * Get the index in the raw stream past the last character in the token. + * + * @return the ending index of the token, which is past the last character + */ int endIndex(); + /** + * Get the length of the token, which is equivalent to <code>endIndex() - startIndex()</code>. + * + * @return the length + */ int length(); + /** + * Get the position of this token, which includes the line number and column number of the first character in the token. + * + * @return the position; never null + */ Position position(); } + /** + * An immutable {@link Token} that implements matching using case-sensitive logic. + */ @Immutable protected class CaseSensitiveToken implements Token { private final int startIndex; @@ -1410,7 +1611,7 @@ } } - protected class CaseSensitiveTokenFactory extends TokenFactory { + public class CaseSensitiveTokenFactory extends TokenFactory { /** * {@inheritDoc} * @@ -1424,7 +1625,7 @@ } } - protected class CaseInsensitiveTokenFactory extends TokenFactory { + public class CaseInsensitiveTokenFactory extends TokenFactory { /** * {@inheritDoc} * @@ -1438,7 +1639,10 @@ } } - protected static final class CharacterArrayStream implements CharacterStream { + /** + * An implementation of {@link CharacterStream} that works with a single character array. + */ + public static final class CharacterArrayStream implements CharacterStream { private final char[] content; private int lastIndex = -1; private final int maxIndex; @@ -1446,7 +1650,7 @@ private int columnNumber = 1; private boolean nextCharMayBeLineFeed; - protected CharacterArrayStream( char[] content ) { + public CharacterArrayStream( char[] content ) { this.content = content; this.maxIndex = content.length - 1; } @@ -1577,6 +1781,16 @@ int nextIndex = lastIndex + 1; return nextIndex <= maxIndex && Character.isWhitespace(content[nextIndex]); } + + /** + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.CharacterStream#isNextValidXmlCharacter() + */ + public boolean isNextValidXmlCharacter() { + int nextIndex = lastIndex + 1; + return nextIndex <= maxIndex && XmlCharacters.isValid(content[nextIndex]); + } } /** Modified: trunk/dna-graph/src/main/java/org/jboss/dna/graph/JcrLexicon.java =================================================================== --- trunk/dna-graph/src/main/java/org/jboss/dna/graph/JcrLexicon.java 2009-09-16 14:39:42 UTC (rev 1225) +++ trunk/dna-graph/src/main/java/org/jboss/dna/graph/JcrLexicon.java 2009-09-16 14:40:56 UTC (rev 1226) @@ -67,6 +67,7 @@ public static final Name PRIMARY_ITEM_NAME = new BasicName(Namespace.URI, "primaryItemName"); public static final Name PROPERTY_DEFINITION = new BasicName(Namespace.URI, "propertyDefinition"); public static final Name PROTECTED = new BasicName(Namespace.URI, "protected"); + public static final Name QUERY_OPERATORS = new BasicName(Namespace.URI, "queryOperators"); public static final Name REQUIRED_PRIMARY_TYPES = new BasicName(Namespace.URI, "requiredPrimaryTypes"); public static final Name REQUIRED_TYPE = new BasicName(Namespace.URI, "requiredType"); public static final Name SAME_NAME_SIBLINGS = new BasicName(Namespace.URI, "sameNameSiblings");