[dna-commits] DNA SVN: r1475 - in trunk: dna-repository/src/main/java/org/jboss/dna/repository/sequencer and 32 other directories.

Author: bcarothers Date: 2009-12-22 16:33:04 -0500 (Tue, 22 Dec 2009) New Revision: 1475 Added: trunk/docs/examples/gettingstarted/sequencers/src/main/resources/fixedWidthFile.txt trunk/docs/examples/gettingstarted/sequencers/src/main/resources/test.csv trunk/docs/reference/src/main/docbook/en-US/content/sequencers/text.xml trunk/extensions/dna-sequencer-text/ trunk/extensions/dna-sequencer-text/.classpath trunk/extensions/dna-sequencer-text/.project trunk/extensions/dna-sequencer-text/pom.xml trunk/extensions/dna-sequencer-text/src/ trunk/extensions/dna-sequencer-text/src/main/ trunk/extensions/dna-sequencer-text/src/main/java/ trunk/extensions/dna-sequencer-text/src/main/java/org/ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/AbstractTextSequencer.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DefaultRowFactory.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DelimitedTextSequencer.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencer.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/RowFactory.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerI18n.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerLexicon.java trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/package-info.java trunk/extensions/dna-sequencer-text/src/main/resources/ trunk/extensions/dna-sequencer-text/src/main/resources/org/ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/TextSequencerI18n.properties trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/sequencer-text.cnd trunk/extensions/dna-sequencer-text/src/test/ trunk/extensions/dna-sequencer-text/src/test/java/ trunk/extensions/dna-sequencer-text/src/test/java/org/ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/DelimitedTextSequencerTest.java trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencerTest.java trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/TextSequencerI18nTest.java trunk/extensions/dna-sequencer-text/src/test/resources/ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFile.csv trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileMissingRecords.csv trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileWithComments.csv trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLinePipeDelimitedFile.csv trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFile.csv trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFileNoTrailingNewLine.csv trunk/extensions/dna-sequencer-text/src/test/resources/fixed/ trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFile.txt trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileMissingRecords.txt trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileWithComments.txt trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFile.txt trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFileNoTrailingNewLine.txt Modified: trunk/dna-repository/src/main/java/org/jboss/dna/repository/sequencer/StreamSequencerAdapter.java trunk/docs/examples/gettingstarted/sequencers/pom.xml trunk/docs/examples/gettingstarted/sequencers/src/main/java/org/jboss/example/dna/sequencer/SequencingClient.java trunk/docs/examples/gettingstarted/sequencers/src/test/java/org/jboss/example/dna/sequencer/SequencingClientTest.java trunk/docs/reference/src/main/docbook/en-US/custom.dtd trunk/docs/reference/src/main/docbook/en-US/master.xml trunk/pom.xml Log: DNA-105 Sequence content from delimited text files Applied a more refined version of the previous DNA-105 patch that includes documentation (incl. the obligatory Reference Guide chapter), integration with the gettingstarted sequencers example, and a maximumLinesToRead property that allows the sequencers to be configured to not sequencer an entire file. The patch also modifes StreamSequencerAdapter.setConfiguration to allow for properties to be set on StreamSequencers through DnaConfiguration. Modified: trunk/dna-repository/src/main/java/org/jboss/dna/repository/sequencer/StreamSequencerAdapter.java =================================================================== --- trunk/dna-repository/src/main/java/org/jboss/dna/repository/sequencer/StreamSequencerAdapter.java 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/dna-repository/src/main/java/org/jboss/dna/repository/sequencer/StreamSequencerAdapter.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -30,8 +30,10 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; import org.jboss.dna.common.collection.Problems; +import org.jboss.dna.common.util.Reflection; import org.jboss.dna.graph.JcrLexicon; import org.jboss.dna.graph.JcrNtLexicon; import org.jboss.dna.graph.Node; @@ -72,6 +74,21 @@ */ public void setConfiguration( SequencerConfig configuration ) { this.configuration = configuration; + + /* + * Try to pass the configured properties through to the stream sequencer + */ + if (configuration.getProperties() != null) { + for (Map.Entry<String, Object> entry : configuration.getProperties().entrySet()) { + // Set the JavaBean-style property on the RepositorySource instance ... + Reflection reflection = new Reflection(streamSequencer.getClass()); + try { + reflection.invokeSetterMethodOnTarget(entry.getKey(), streamSequencer, entry.getValue()); + } catch (Exception ignore) { + // It's possible that these properties weren't intended for the stream sequencer anyway + } + } + } } /** @@ -264,8 +281,7 @@ Path path = factories.getPathFactory().create(input.getLocation().getPath()); Set<org.jboss.dna.graph.property.Property> props = new HashSet<org.jboss.dna.graph.property.Property>( - input.getPropertiesByName() - .values()); + input.getPropertiesByName().values()); props = Collections.unmodifiableSet(props); String mimeType = getMimeType(context, sequencedProperty, path.getLastSegment().getName().getLocalName()); return new StreamSequencerContext(context.getExecutionContext(), path, props, mimeType, problems); Modified: trunk/docs/examples/gettingstarted/sequencers/pom.xml =================================================================== --- trunk/docs/examples/gettingstarted/sequencers/pom.xml 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/docs/examples/gettingstarted/sequencers/pom.xml 2009-12-22 21:33:04 UTC (rev 1475) @@ -48,6 +48,12 @@ </dependency> <dependency> <groupId>org.jboss.dna</groupId> + <artifactId>dna-sequencer-text</artifactId> + <version>${project.version}</version> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.jboss.dna</groupId> <artifactId>dna-sequencer-zip</artifactId> <version>${project.version}</version> <scope>test</scope> Modified: trunk/docs/examples/gettingstarted/sequencers/src/main/java/org/jboss/example/dna/sequencer/SequencingClient.java =================================================================== --- trunk/docs/examples/gettingstarted/sequencers/src/main/java/org/jboss/example/dna/sequencer/SequencingClient.java 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/docs/examples/gettingstarted/sequencers/src/main/java/org/jboss/example/dna/sequencer/SequencingClient.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -96,7 +96,23 @@ .setDescription("Sequences Java files to extract the AST structure of the Java source code") .sequencingFrom("//(*.java[*])/jcr:content[@jcr:data]") .andOutputtingTo("/java/$1"); - + // Set up the CSV file sequencer ... + config.sequencer("CSV Sequencer") + .usingClass("org.jboss.dna.sequencer.text.DelimitedTextSequencer") + .loadedFromClasspath() + .setDescription("Sequences CSV files to extract the contents") + .sequencingFrom("//(*.csv[*])/jcr:content[@jcr:data]") + .andOutputtingTo("/csv/$1"); + // Set up the fixed width file sequencer ... + config.sequencer("Fixed Width Sequencer") + .usingClass("org.jboss.dna.sequencer.text.FixedWidthTextSequencer") + .loadedFromClasspath() + .setDescription("Sequences fixed width files to extract the contents") + .setProperty("commentMarker", "#") + .setProperty("columnStartPositions", new int[] { 10, 20, 30, 40}) + .sequencingFrom("//(*.txt[*])/jcr:content[@jcr:data]") + .andOutputtingTo("/txt/$1"); + // Now start the client and tell it which repository and workspace to use ... SequencingClient client = new SequencingClient(config, repositoryId, workspaceName); client.setUserInterface(new ConsoleInput(client)); @@ -186,7 +202,7 @@ String mimeType = getMimeType(url); if (mimeType == null) { - System.err.println("Could not determine mime type for file. Cancelling upload."); + System.err.println("Could not determine mime type for file " + url + ". Cancelling upload."); return; } @@ -266,6 +282,67 @@ } } + if (root.hasNode("txt")) { + LinkedList<Node> nodesToVisit = new LinkedList<Node>(); + + for (NodeIterator i = root.getNode("txt").getNodes(); i.hasNext();) { + nodesToVisit.addLast(i.nextNode()); + } + + while (!nodesToVisit.isEmpty()) { + Node node = nodesToVisit.remove(); + Properties props = new Properties(); + int rowCount = 0; + int colCount = 0; + + for (NodeIterator rowIter = node.getNodes(); rowIter.hasNext();) { + Node rowNode = rowIter.nextNode(); + + rowCount++; + colCount = 0; + + for (NodeIterator colIter = rowNode.getNodes(); colIter.hasNext();) { + colIter.nextNode(); + colCount++; + } + } + props.put("rows", String.valueOf(rowCount)); + props.put("columns", String.valueOf(colCount)); + + infos.add(new ContentInfo(node.getPath(), node.getName(), props)); + } + } + if (root.hasNode("csv")) { + LinkedList<Node> nodesToVisit = new LinkedList<Node>(); + + for (NodeIterator i = root.getNode("csv").getNodes(); i.hasNext();) { + nodesToVisit.addLast(i.nextNode()); + } + + while (!nodesToVisit.isEmpty()) { + Node node = nodesToVisit.remove(); + Properties props = new Properties(); + int rowCount = 0; + int colCount = 0; + + for (NodeIterator rowIter = node.getNodes(); rowIter.hasNext();) { + Node rowNode = rowIter.nextNode(); + + rowCount++; + colCount = 0; + + for (NodeIterator colIter = rowNode.getNodes(); colIter.hasNext();) { + colIter.nextNode(); + colCount++; + } + } + props.put("rows", String.valueOf(rowCount)); + props.put("columns", String.valueOf(colCount)); + + infos.add(new ContentInfo(node.getPath(), node.getName(), props)); + } + } + if (root.hasNode("java")) { Map<String, List<Properties>> tree = new TreeMap<String, List<Properties>>(); // Find the compilation unit node ... @@ -466,6 +543,8 @@ if (filename.endsWith(".mp3")) return "audio/mpeg"; if (filename.endsWith(".jar")) return "application/java-archive"; if (filename.endsWith(".java")) return "text/x-java-source"; + if (filename.endsWith(".csv")) return "text/csv"; + if (filename.endsWith(".txt")) return "text/plain"; return null; } Added: trunk/docs/examples/gettingstarted/sequencers/src/main/resources/fixedWidthFile.txt =================================================================== --- trunk/docs/examples/gettingstarted/sequencers/src/main/resources/fixedWidthFile.txt (rev 0) +++ trunk/docs/examples/gettingstarted/sequencers/src/main/resources/fixedWidthFile.txt 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,2 @@ +00000000001111111111222222222233333333334444444444 +This is a flat file that will be parsed into cols. Property changes on: trunk/docs/examples/gettingstarted/sequencers/src/main/resources/fixedWidthFile.txt ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/docs/examples/gettingstarted/sequencers/src/main/resources/test.csv =================================================================== --- trunk/docs/examples/gettingstarted/sequencers/src/main/resources/test.csv (rev 0) +++ trunk/docs/examples/gettingstarted/sequencers/src/main/resources/test.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,2 @@ +This,is,a,comma-delimited,file +It,has,two,rows,whee! \ No newline at end of file Modified: trunk/docs/examples/gettingstarted/sequencers/src/test/java/org/jboss/example/dna/sequencer/SequencingClientTest.java =================================================================== --- trunk/docs/examples/gettingstarted/sequencers/src/test/java/org/jboss/example/dna/sequencer/SequencingClientTest.java 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/docs/examples/gettingstarted/sequencers/src/test/java/org/jboss/example/dna/sequencer/SequencingClientTest.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -53,6 +53,8 @@ private URL mp3Url; private URL jarUrl; private URL javaSourceUrl; + private URL csvUrl; + private URL fixedWidthFileUrl; private SequencingClient client; @Before @@ -62,6 +64,9 @@ this.jpegImageUrl = Thread.currentThread().getContextClassLoader().getResource("caution.jpg"); this.mp3Url = Thread.currentThread().getContextClassLoader().getResource("sample1.mp3"); this.jarUrl = Thread.currentThread().getContextClassLoader().getResource("test.jar"); + this.csvUrl = Thread.currentThread().getContextClassLoader().getResource("test.csv"); + this.fixedWidthFileUrl = Thread.currentThread().getContextClassLoader().getResource("fixedWidthFile.txt"); + // Get the URL of source (MySource.java), that have to be sequencing this.javaSourceUrl = FileUtil.convertFileToURL("workspace/project1/src/org/acme/MySource.java"); @@ -103,7 +108,23 @@ .setDescription("Sequences Java files to extract the AST structure of the Java source code") .sequencingFrom("//(*.java[*])/jcr:content[@jcr:data]") .andOutputtingTo("/java/$1"); - + // Set up the CSV file sequencer ... + config.sequencer("CSV Sequencer") + .usingClass("org.jboss.dna.sequencer.text.DelimitedTextSequencer") + .loadedFromClasspath() + .setDescription("Sequences CSV files to extract the contents") + .sequencingFrom("//(*.csv[*])/jcr:content[@jcr:data]") + .andOutputtingTo("/csv/$1"); + // Set up the fixed width file sequencer ... + config.sequencer("Fixed Width Sequencer") + .usingClass("org.jboss.dna.sequencer.text.FixedWidthTextSequencer") + .loadedFromClasspath() + .setDescription("Sequences fixed width files to extract the contents") + .setProperty("commentMarker", "#") + .setProperty("columnStartPositions", new int[] { 10, 20, 30, 40}) + .sequencingFrom("//(*.txt[*])/jcr:content[@jcr:data]") + .andOutputtingTo("/txt/$1"); + // Now start the client and tell it which repository and workspace to use ... client = new SequencingClient(config, repositoryId, workspaceName); } @@ -193,6 +214,32 @@ client.search(); } + @Test + public void shouldUploadAndSequenceDelimitedFile() throws Exception { + client.setUserInterface(new MockUserInterface(this.csvUrl, "/a/b/test.csv", 1)); + client.startRepository(); + client.uploadFile(); + + waitUntilSequencedNodesIs(1); + + // The sequencers should have run, so perform the search. + // The mock user interface checks the results. + client.search(); + } + + @Test + public void shouldUploadAndSequenceFixedWidthFile() throws Exception { + client.setUserInterface(new MockUserInterface(this.fixedWidthFileUrl, "/a/b/fixedWidthFile.txt", 1)); + client.startRepository(); + client.uploadFile(); + + waitUntilSequencedNodesIs(1); + + // The sequencers should have run, so perform the search. + // The mock user interface checks the results. + client.search(); + } + protected void waitUntilSequencedNodesIs( int totalNumberOfNodesSequenced ) throws InterruptedException { // check 50 times, waiting 0.1 seconds between (for a total of 5 seconds max) ... long numFound = 0; Added: trunk/docs/reference/src/main/docbook/en-US/content/sequencers/text.xml =================================================================== --- trunk/docs/reference/src/main/docbook/en-US/content/sequencers/text.xml (rev 0) +++ trunk/docs/reference/src/main/docbook/en-US/content/sequencers/text.xml 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,192 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ JBoss DNA (http://www.jboss.org/dna) + ~ + ~ See the COPYRIGHT.txt file distributed with this work for information + ~ regarding copyright ownership. Some portions may be licensed + ~ to Red Hat, Inc. under one or more contributor license agreements. + ~ See the AUTHORS.txt file in the distribution for a full listing of + ~ individual contributors. + ~ + ~ JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + ~ is licensed to you under the terms of the GNU Lesser General Public License as + ~ published by the Free Software Foundation; either version 2.1 of + ~ the License, or (at your option) any later version. + ~ + ~ JBoss DNA is distributed in the hope that it will be useful, + ~ but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + ~ or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + ~ for more details. + ~ + ~ You should have received a copy of the GNU Lesser General Public License + ~ along with this distribution; if not, write to: + ~ Free Software Foundation, Inc. + ~ 51 Franklin Street, Fifth Floor + ~ Boston, MA 02110-1301 USA + --> +<!DOCTYPE preface PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ +<!ENTITY % CustomDTD SYSTEM "../../custom.dtd"> +%CustomDTD; +]> +<chapter id="text-sequencer"> + <title>Text Sequencers</title> + <para> + The text sequencers extract data from text streams. There are separate sequencers for character-delimited sequencing and fixed + width sequencing, but both treat the incoming text stream as a series of rows (separated by line-terminators, as defined in + &BufferedReader;.readLine() with each row consisting of one or more columns. As noted above, each text sequencer provides its + own mechanism for splitting the row into columns. + </para> + <para> + The &AbstractTextSequencer; class provides a number of JavaBean properties that are common to both of the concrete text sequencer classes: + </para> + <table frame='all'> + <title>&AbstractTextSequencer; properties</title> + <tgroup cols='2' align='left' colsep='1' rowsep='1'> + <colspec colname='c1' colwidth="1*"/> + <colspec colname='c2' colwidth="1*"/> + <thead> + <row> + <entry>Property</entry> + <entry>Description</entry> + </row> + </thead> + <tbody> + <row> + <entry>commentMarker</entry> + <entry> + Optional property that, if set, indicates that any line beginning with <emphasis>exactly this string</emphasis> should + be treated as a comment and should not be processed further. If this value is null, then all lines will be sequenced. + The default value for this property is null. + </entry> + </row> + <row> + <entry>maximumLinesToRead</entry> + <entry> + Optional property that, if set, limits the number of lines that will be read during sequencing. Additional lines will be + ignored. If this value is non-positive, all lines will be read and sequenced. Comment lines are not counted towards this total. + The default value of this property is -1 (indicating that all lines should be read and sequenced). + </entry> + </row> + <row> + <entry>rowFactoryClassName</entry> + <entry> + Optional property that, if set, provides the name of a class that provides a custom implementation of the &RowFactory; interface. + This class must have a no-argument, public constructor. If set, an instance of this class will be created each time that the sequencer + sequences an input stream and will be used to provide the output structure of the graph. If this property is set to null, + a default implementation will be used. The default value of this property is null. + </entry> + </row> + </tbody> + </tgroup> + </table> + <para> + The default row factory creates one node in the output location for each row sequenced from the source and adds each column with the row as a + child node of the row node. The output graph takes the following form (all nodes have primary type <code>nt:unstructured</code>: + </para> + <programlisting> + &lt;graph root&gt; + + text:row[1] + | + text:column[1] (jcr:mixinTypes = text:column, text:data = &lt;column1 data&gt;) + | + ... + | + text:column[n] (jcr:mixinTypes = text:column, text:data = &lt;columnN data&gt;) + + ... + + text:row[m] + + text:column[1] (jcr:mixinTypes = text:column, text:data = &lt;column1 data&gt;) + + ... + + text:column[n] (jcr:mixinTypes = text:column, text:data = &lt;columnN data&gt;) + + </programlisting> + <sect1> + <title>Delimited Text Sequencer</title> + <para> + The &DelimitedTextSequencer; splits rows into columns based on a regular expression pattern. Although the default pattern is a comma, + any regular expression can be provided allowing for more sophisticated splitting patterns. + </para> + <para> + The &DelimitedTextSequencer; class provides an additional JavaBean property to override the default regular expression pattern: + </para> + <table frame='all'> + <title>&DelimitedTextSequencer; properties</title> + <tgroup cols='2' align='left' colsep='1' rowsep='1'> + <colspec colname='c1' colwidth="1*"/> + <colspec colname='c2' colwidth="1*"/> + <thead> + <row> + <entry>Property</entry> + <entry>Description</entry> + </row> + </thead> + <tbody> + <row> + <entry>splitPattern</entry> + <entry> + Optional property that, if set, sets the regular expression pattern that is used to split each row into columns. This property + may not be set to null and defaults to ",". + </entry> + </row> + </tbody> + </tgroup> + </table> + <para> + To use this sequencer, simply include the <code>dna-sequencer-text</code> JAR + in your application and configure the &JcrConfiguration; to use this sequencer using something similar to: + </para> + <programlisting> +JcrConfiguration config = ... + +config.sequencer("Delimited Text Sequencer") + .usingClass("org.jboss.dna.sequencer.text.DelimitedTextSequencer") + .loadedFromClasspath() + .setDescription("Sequences delimited files to extract values") + .sequencingFrom("//(*.(txt)[*])/jcr:content[@jcr:data]") + .setProperty("splitPattern", "|") + .andOutputtingTo("/txt/$1"); + </programlisting> + </sect1> + <sect1> + <title>Fixed Width Text Sequencer</title> + <para> + The &FixedWidthTextSequencer; splits rows into columns based on predefined positions. The default setting is to have a single + column per row. It also provides an additional JavaBean property to override the default start positions for each column. + </para> + <table frame='all'> + <title>&FixedWidthTextSequencer; properties</title> + <tgroup cols='2' align='left' colsep='1' rowsep='1'> + <colspec colname='c1' colwidth="1*"/> + <colspec colname='c2' colwidth="1*"/> + <thead> + <row> + <entry>Property</entry> + <entry>Description</entry> + </row> + </thead> + <tbody> + <row> + <entry>columnStartPositions</entry> + <entry> + Optional property that, if set, provides the start position of each column after the first. The start + positions are concatenated into a single, comma-delimited string. The default value is the empty string + (implying that each row should be treated as a single column). This property may not be set to null. There + is an implicit column start position of 0 that never needs to be specified. + </entry> + </row> + </tbody> + </tgroup> + </table> + <para> + To use this sequencer, simply include the <code>dna-sequencer-text</code> JAR + in your application and configure the &JcrConfiguration; to use this sequencer using something similar to: + </para> + <programlisting> +JcrConfiguration config = ... + +config.sequencer("Fixed Width Text Sequencer") + .usingClass("org.jboss.dna.sequencer.text.FixedWidthTextSequencer") + .loadedFromClasspath() + .setDescription("Sequences fixed width files to extract values") + .sequencingFrom("//(*.(txt)[*])/jcr:content[@jcr:data]") + .setProperty("columnStartPositions", "3,6,15") + .andOutputtingTo("/txt/$1"); + </programlisting> + </sect1> +</chapter> \ No newline at end of file Property changes on: trunk/docs/reference/src/main/docbook/en-US/content/sequencers/text.xml ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Modified: trunk/docs/reference/src/main/docbook/en-US/custom.dtd =================================================================== --- trunk/docs/reference/src/main/docbook/en-US/custom.dtd 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/docs/reference/src/main/docbook/en-US/custom.dtd 2009-12-22 21:33:04 UTC (rev 1475) @@ -35,6 +35,7 @@ <!ENTITY String "<ulink url='&Java;java/lang/String.html'><interface>String</interface></ulink>"> <!ENTITY File "<ulink url='&Java;java/io/File.html'><classname>File</classname></ulink>"> +<!ENTITY BufferedReader "<ulink url='&Java;java/io/BufferedReader.html'><classname>BufferedReader</classname></ulink>"> <!ENTITY URL "<ulink url='&Java;java/net/URL.html'><classname>URL</classname></ulink>"> <!ENTITY URI "<ulink url='&Java;java/net/URL.html'><classname>URI</classname></ulink>"> <!ENTITY InputStream "<ulink url='&Java;java/io/InputStream.html'><interface>InputStream</interface></ulink>"> @@ -215,3 +216,8 @@ <!ENTITY DnaJcrDeployer "<ulink url='&API;web/jcr/rest/DnaJcrDeployer.html'><classname>DnaJcrDeployer</classname></ulink>"> <!ENTITY RepositoryProvider "<ulink url='&API;web/jcr/rest/spi/RepositoryProvider.html'><classname>RepositoryProvider</classname></ulink>"> +<!ENTITY AbstractTextSequencer "<ulink url='&API;sequencer/text/AbstractTextSequencer.html'><classname>AbstractTextSequencer</classname></ulink>"> +<!ENTITY DelimitedTextSequencer "<ulink url='&API;sequencer/text/DelimitedTextSequencer'><classname>DelimitedTextSequencer</classname></ulink>"> +<!ENTITY FixedWidthTextSequencer "<ulink url='&API;sequencer/text/FixedWidthTextSequencer'><classname>FixedWidthTextSequencer</classname></ulink>"> +<!ENTITY RowFactory "<ulink url='&API;sequencer/text/RowFactory'><classname>RowFactory</classname></ulink>"> + Modified: trunk/docs/reference/src/main/docbook/en-US/master.xml =================================================================== --- trunk/docs/reference/src/main/docbook/en-US/master.xml 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/docs/reference/src/main/docbook/en-US/master.xml 2009-12-22 21:33:04 UTC (rev 1475) @@ -135,6 +135,7 @@ <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="content/sequencers/image.xml"/> <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="content/sequencers/mp3.xml"/> <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="content/sequencers/ddl.xml"/> + <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="content/sequencers/text.xml"/> </part> <part id="provied-mime-type-detectors-part"> <title>MIME Type Detector Library</title> Added: trunk/extensions/dna-sequencer-text/.classpath =================================================================== --- trunk/extensions/dna-sequencer-text/.classpath (rev 0) +++ trunk/extensions/dna-sequencer-text/.classpath 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src/main/java"/> + <classpathentry kind="src" path="src/main/resources"/> + <classpathentry kind="src" output="target/test-classes" path="src/test/java"/> + <classpathentry kind="src" output="target/test-classes" path="src/test/resources"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> + <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/> + <classpathentry kind="output" path="target/classes"/> +</classpath> \ No newline at end of file Added: trunk/extensions/dna-sequencer-text/.project =================================================================== --- trunk/extensions/dna-sequencer-text/.project (rev 0) +++ trunk/extensions/dna-sequencer-text/.project 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8"?> +<projectDescription> + <name>dna-sequencer-text</name> + <comment></comment> + <projects> + </projects> + <buildSpec> + <buildCommand> + <name>org.eclipse.jdt.core.javabuilder</name> + <arguments> + </arguments> + </buildCommand> + <buildCommand> + <name>org.maven.ide.eclipse.maven2Builder</name> + <arguments> + </arguments> + </buildCommand> + </buildSpec> + <natures> + <nature>org.eclipse.jdt.core.javanature</nature> + <nature>org.maven.ide.eclipse.maven2Nature</nature> + </natures> +</projectDescription> Added: trunk/extensions/dna-sequencer-text/pom.xml =================================================================== --- trunk/extensions/dna-sequencer-text/pom.xml (rev 0) +++ trunk/extensions/dna-sequencer-text/pom.xml 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,60 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>dna</artifactId> + <groupId>org.jboss.dna</groupId> + <version>0.7-SNAPSHOT</version> + <relativePath>../..</relativePath> + </parent> + <!-- The groupId and version values are inherited from parent --> + <artifactId>dna-sequencer-text</artifactId> + <packaging>jar</packaging> + <name>JBoss Text File Sequencer</name> + <description>JBoss DNA Sequencer that processes fixed width and delimited text files</description> + <url>http://labs.jboss.org/dna</url> + <dependencies> + <dependency> + <groupId>org.jboss.dna</groupId> + <artifactId>dna-common</artifactId> + </dependency> + <dependency> + <groupId>org.jboss.dna</groupId> + <artifactId>dna-common</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.jboss.dna</groupId> + <artifactId>dna-graph</artifactId> + </dependency> + <dependency> + <groupId>org.jboss.dna</groupId> + <artifactId>dna-graph</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <!-- + Logging (require SLF4J API for compiling, but use Log4J and its SLF4J binding for testing) + --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file Property changes on: trunk/extensions/dna-sequencer-text/pom.xml ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/AbstractTextSequencer.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/AbstractTextSequencer.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/AbstractTextSequencer.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,169 @@ +package org.jboss.dna.sequencer.text; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import net.jcip.annotations.ThreadSafe; +import org.jboss.dna.common.util.Logger; +import org.jboss.dna.graph.sequencer.SequencerOutput; +import org.jboss.dna.graph.sequencer.StreamSequencer; +import org.jboss.dna.graph.sequencer.StreamSequencerContext; + +/** + * The base class for the text sequencers. This class treats the text to be sequenced as a series of rows, with each row delimited + * by a line terminator. Concrete subclasses provide their own mechanisms for splitting a row of data into a series of columns. + * <p> + * This class provides some fundamental capabilities, including the ability to set a {@link #setCommentMarker(String) comment + * marker}, {@link #setMaximumLinesToRead(int) limit the number of lines} to be read from a file, and + * {@link #setRowFactoryClassName(String) provide custom transformations} from the sets of columns to the graph structure. + * </p> + */ +@ThreadSafe +public abstract class AbstractTextSequencer implements StreamSequencer { + + private final Logger logger = Logger.getLogger(getClass()); + private String rowFactoryClassName = null; + private String commentMarker = null; + private int maximumLinesToRead = -1; + + /** + * {@inheritDoc} + */ + public void sequence( InputStream stream, + SequencerOutput output, + StreamSequencerContext context ) { + BufferedReader reader = null; + String line = null; + RowFactory rowFactory = null; + String rowFactoryClassName = this.rowFactoryClassName; + String commentMarker = this.commentMarker; + int maxLinesToRead = this.maximumLinesToRead; + + int rowCount = 0; + + try { + rowFactory = createRowFactory(rowFactoryClassName); + } catch (Exception ex) { + // This really shouldn't be able to happen, as we test-instantiate the class before it is set + throw new IllegalStateException(TextSequencerI18n.couldNotInstantiateRowFactory.text(rowFactoryClassName)); + } + + try { + reader = new BufferedReader(new InputStreamReader(stream)); + + while ((line = reader.readLine()) != null) { + if (commentMarker != null && line.startsWith(this.commentMarker)) continue; + if ((maxLinesToRead > 0) && (++rowCount > maxLinesToRead)) return; + String[] columns = parseLine(line); + + rowFactory.recordRow(context, output, columns); + } + + } catch (IOException ioe) { + logger.error(ioe, TextSequencerI18n.errorReadingLine, context.getInputPath()); + + } finally { + try { + if (reader != null) reader.close(); + } catch (Exception ignore) { + } + } + + } + + /** + * Sets the comment marker to use. Any line that begins with the comment marker will be ignored and will not be counted as a + * read line for the purposes of the {@link #getMaximumLinesToRead() maximum line limitation}. + * + * @param commentMarker the string that indicates that the line is a comment and should be ignored; null indicates that there + * is no comment marker + */ + public void setCommentMarker( String commentMarker ) { + this.commentMarker = commentMarker; + } + + /** + * @return the current comment marker; may be null + */ + public String getCommentMarker() { + return commentMarker; + } + + /** + * @return the maximum number of lines to read when sequencing; non-positive numbers indicate that all lines should be read + * and sequenced + */ + public int getMaximumLinesToRead() { + return maximumLinesToRead; + } + + /** + * Sets the maximum number of lines to read. When this number is reached during the sequencing of any particular stream, the + * stream will be closed and remaining lines (if any) will be ignored. {@link #setCommentMarker(String) Comment lines} do not + * count towards the number of lines read. + * + * @param maximumLinesToRead the maximum number of lines to read; a non-positive number indicates that all lines should be + * read and sequenced. + */ + public void setMaximumLinesToRead( int maximumLinesToRead ) { + this.maximumLinesToRead = maximumLinesToRead; + } + + /** + * @return the current row factory class name; may not be null + */ + public String getRowFactoryClassName() { + return rowFactoryClassName; + } + + /** + * Sets the custom row factory class name. This method attempts to instantiate an instance of the custom {@link RowFactory} + * class prior to modifying the row factory class name to ensure that the new value represents a valid implementation. + * + * @param rowFactoryClassName the fully-qualified class name of the new custom row factory implementation; null indicates that + * {@link DefaultRowFactory the default row factory} should be used. + * @throws ClassNotFoundException if the the named row factory class cannot be located + * @throws IllegalAccessException if the row factory class or its nullary constructor is not accessible. + * @throws InstantiationException if the row factory represents an abstract class, an interface, an array class, a primitive + * type, or void; or if the class has no nullary constructor; or if the instantiation fails for some other reason. + * @throws ClassCastException if the instantiated row factory does not implement the {@link RowFactory} interface + */ + public synchronized void setRowFactoryClassName( String rowFactoryClassName ) + throws ClassNotFoundException, IllegalAccessException, InstantiationException { + // Make sure that it's going to work + createRowFactory(rowFactoryClassName); + this.rowFactoryClassName = rowFactoryClassName; + } + + /** + * Parse the given row into its constituent columns. + * + * @param row the row to be parsed + * @return an array of columns; never null + */ + protected abstract String[] parseLine( String row ); + + /** + * Creates an instance of the {@link #getRowFactoryClassName() row factory} configured for this sequencer. + * + * @param className the name of the class to configure; null indicates that the {@link DefaultRowFactory default row factory} + * should be used. + * @return an implementation of the named class; never null + * @throws ClassNotFoundException if the the named row factory class cannot be located + * @throws IllegalAccessException if the row factory class or its nullary constructor is not accessible. + * @throws InstantiationException if the row factory represents an abstract class, an interface, an array class, a primitive + * type, or void; or if the class has no nullary constructor; or if the instantiation fails for some other reason. + */ + protected synchronized RowFactory createRowFactory( String className ) + throws ClassNotFoundException, IllegalAccessException, InstantiationException { + if (this.rowFactoryClassName == null) { + return new DefaultRowFactory(); + } + + Class<?> rowFactoryClass = Class.forName(this.rowFactoryClassName); + RowFactory rowFactory = (RowFactory)rowFactoryClass.newInstance(); + + return rowFactory; + } +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/AbstractTextSequencer.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DefaultRowFactory.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DefaultRowFactory.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DefaultRowFactory.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,57 @@ +package org.jboss.dna.sequencer.text; + +import net.jcip.annotations.NotThreadSafe; +import org.jboss.dna.graph.JcrLexicon; +import org.jboss.dna.graph.JcrNtLexicon; +import org.jboss.dna.graph.property.PathFactory; +import org.jboss.dna.graph.property.Path.Segment; +import org.jboss.dna.graph.sequencer.SequencerOutput; +import org.jboss.dna.graph.sequencer.StreamSequencerContext; + +/** + * A default implementation of the {@link RowFactory} class. This class records rows in the following subgraph: + * <pre> + * &lt;graph root&gt; + * + text:row[1] (jcr:primaryType = nt:unstructured) + * | + text:column[1] (jcr:primaryType = nt:unstructured, jcr:mixinTypes = text:column, text:data = <column1 data>) + * | + ... + * | + text:column[n] (jcr:primaryType = nt:unstructured, jcr:mixinTypes = text:column, text:data = <columnN data>) + * + ... + * + text:row[m] (jcr:primaryType = nt:unstructured) + * + text:column[1] (jcr:primaryType = nt:unstructured, jcr:mixinTypes = text:column, text:data = <column1 data>) + * + ... + * + text:column[n] (jcr:primaryType = nt:unstructured, jcr:mixinTypes = text:column, text:data = <columnN data>) + * </pre> + */ +@NotThreadSafe +public class DefaultRowFactory implements RowFactory { + + private int rowNumber = 1; + + @Override + public void recordRow( StreamSequencerContext context, + SequencerOutput output, + String[] columns ) { + + int columnNumber = 1; + PathFactory pathFactory = context.getValueFactories().getPathFactory(); + + Segment rowSegment = pathFactory.createSegment(TextSequencerLexicon.ROW, rowNumber); + output.setProperty(pathFactory.createRelativePath(rowSegment), JcrLexicon.PRIMARY_TYPE, JcrNtLexicon.UNSTRUCTURED); + + for (String column : columns) { + Segment columnSegment = pathFactory.createSegment(TextSequencerLexicon.COLUMN, columnNumber++); + + output.setProperty(pathFactory.createRelativePath(rowSegment, columnSegment), + JcrLexicon.PRIMARY_TYPE, + JcrNtLexicon.UNSTRUCTURED); + output.setProperty(pathFactory.createRelativePath(rowSegment, columnSegment), + JcrLexicon.MIXIN_TYPES, + TextSequencerLexicon.COLUMN); + output.setProperty(pathFactory.createRelativePath(rowSegment, columnSegment), TextSequencerLexicon.DATA, column); + } + + rowNumber++; + } + +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DefaultRowFactory.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DelimitedTextSequencer.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DelimitedTextSequencer.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DelimitedTextSequencer.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,36 @@ +package org.jboss.dna.sequencer.text; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import org.jboss.dna.common.util.CheckArg; + +/** + * An text sequencer implementation that uses a {@link #setSplitPattern(String) regular-expression pattern} to split incoming rows + * into columns. By default, this class uses the pattern {@code ","} to parse files on commas. + * + * @see AbstractTextSequencer + * @see Pattern + */ +public class DelimitedTextSequencer extends AbstractTextSequencer { + + private Pattern splitPattern = Pattern.compile(","); + + /** + * Sets the regular expression to use to split incoming rows. + * + * @param regularExpression the regular expression to use to split incoming rows; may not be null. + * @throws PatternSyntaxException if {@code regularExpression} does not represent a valid regular expression that can be + * {@link Pattern#compile(String) compiled into a pattern}. + */ + public void setSplitPattern( String regularExpression ) throws PatternSyntaxException { + CheckArg.isNotNull(regularExpression, "regularExpression"); + + splitPattern = Pattern.compile(regularExpression); + } + + @Override + protected String[] parseLine( String line ) { + return splitPattern.split(line); + } + +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/DelimitedTextSequencer.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencer.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencer.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencer.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,112 @@ +package org.jboss.dna.sequencer.text; + +import java.util.Arrays; +import org.jboss.dna.common.util.CheckArg; + +/** + * An text sequencer implementation that uses a {@link #setColumnStartPositions(String) list of column numbers} to split incoming + * rows into fixed-width columns. By default, this class treats each row as a single column. There is an implicit column start + * index of 0 for the first column. + * + * @see AbstractTextSequencer + */ +public class FixedWidthTextSequencer extends AbstractTextSequencer { + + private int[] columnStartPositions = new int[] {}; + + /** + * Set the column start positions. The column start positions are 0-based. Everything before the first start position is + * treated as the first column. + * <p> + * As an example, if the column start positions were {3, 6, 15} and the incoming stream was: + * + * <pre> + * 1 2 + * 012345678901234567890 + * supercallifragilistic + * expialidocious + * </pre> + * This sequencer would return the following rows: + * <pre> + * row 1: "sup", "erc", "allifragi", "listic" + * row 2: "exp:, "ial", "idocious" + * </pre> + * + * Note that there are only three columns returned in the second row, as there were not enough characters to reach the third + * start position. + * </p> + * + * @param columnStartPositions the column startPositions; may not be null + */ + public void setColumnStartPositions( int[] columnStartPositions ) { + CheckArg.isNotNull(columnStartPositions, "columnStartPositions"); + + this.columnStartPositions = columnStartPositions; + Arrays.sort(columnStartPositions); + } + + /** + * Set the column start positions from a list of column start positions concatenated into a single, comma-delimited string. + * + * @param commaDelimitedColumnStartPositions a list of column start positions concatenated into a single, comma-delimited + * string; may not be null + * @see #setColumnStartPositions(int[]) + */ + public void setColumnStartPositions( String commaDelimitedColumnStartPositions ) { + CheckArg.isNotNull(commaDelimitedColumnStartPositions, "commaDelimitedColumnStartPositions"); + + String[] stringStartPositions = commaDelimitedColumnStartPositions.split(","); + int[] columnStartPositions = new int[stringStartPositions.length]; + + for (int i = 0; i < stringStartPositions.length; i++) { + columnStartPositions[i] = Integer.valueOf(stringStartPositions[i]); + } + + setColumnStartPositions(columnStartPositions); + } + + @Override + protected String[] parseLine( String line ) { + assert line != null; + + int columnCount = columnStartPositions.length + 1; + int currentPos = 0; + + String[] columns = new String[columnCount]; + + for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { + int endPos = columnIndex >= columnCount - 1 ? Integer.MAX_VALUE : columnStartPositions[columnIndex]; + String chunk = parseColumn(line, currentPos, endPos); + + // The line was shorter than expected + if (chunk == null) { + assert columnIndex > 0 : "parseColumn failed to return the first column in string " + line; + + String[] truncatedColumns = new String[columnIndex - 1]; + System.arraycopy(columns, 0, truncatedColumns, 0, columnIndex - 1); + return truncatedColumns; + } + columns[columnIndex] = chunk; + currentPos = endPos; + } + + return columns; + } + + private String parseColumn( String line, + int startPos, + int endPos ) { + int length = line.length(); + + if (length < startPos) { + return null; + } + + if (length < endPos) { + return line.substring(startPos); + } + + return line.substring(startPos, endPos); + } + +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencer.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/RowFactory.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/RowFactory.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/RowFactory.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,34 @@ +package org.jboss.dna.sequencer.text; + +import net.jcip.annotations.NotThreadSafe; +import org.jboss.dna.graph.sequencer.SequencerOutput; +import org.jboss.dna.graph.sequencer.StreamSequencer; +import org.jboss.dna.graph.sequencer.StreamSequencerContext; + +/** + * A simple interface that allows an implementer to control how rows in a text file are mapped to properties (including primary + * and mixin types) in the graph. + * <p> + * Implementations of this class must provide a public, no-argument constructor. + * </p> + * <p> + * To use, supply the implementation class name to a {@link AbstractTextSequencer} object. New instances are created for each + * {@link StreamSequencer#sequence(java.io.InputStream, SequencerOutput, StreamSequencerContext)}, so implementations of this + * interface need not be thread-safe. + * </p> + */ +@NotThreadSafe +public interface RowFactory { + + /** + * Records a row using the provided {@link SequencerOutput} instance. + * + * @param context the sequencer context + * @param output the {@link StreamSequencer} output + * @param columns the columns that could be parsed out for the row + */ + void recordRow( StreamSequencerContext context, + SequencerOutput output, + String[] columns ); + +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/RowFactory.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerI18n.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerI18n.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerI18n.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,57 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. +* See the AUTHORS.txt file in the distribution for a full listing of +* individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.dna.sequencer.text; + +import java.util.Locale; +import java.util.Set; +import org.jboss.dna.common.i18n.I18n; + +/** + * The internationalized string constants for the <code>org.jboss.dna.sequencer.text</code> packages. + */ +public final class TextSequencerI18n { + + public static I18n errorReadingLine; + public static I18n couldNotInstantiateRowFactory; + + static { + try { + I18n.initialize(TextSequencerI18n.class); + } catch (final Exception err) { + System.err.println(err); + } + } + + public static Set<Locale> getLocalizationProblemLocales() { + return I18n.getLocalizationProblemLocales(TextSequencerI18n.class); + } + + public static Set<String> getLocalizationProblems() { + return I18n.getLocalizationProblems(TextSequencerI18n.class); + } + + public static Set<String> getLocalizationProblems( Locale locale ) { + return I18n.getLocalizationProblems(TextSequencerI18n.class, locale); + } +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerI18n.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerLexicon.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerLexicon.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerLexicon.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,42 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.dna.sequencer.text; + +import org.jboss.dna.graph.property.Name; +import org.jboss.dna.graph.property.basic.BasicName; + +/** + * The namespace and property names used within a {@link AbstractTextSequencer} to store internal information. + */ +public class TextSequencerLexicon { + + public static class Namespace { + public static final String URI = "http://www.jboss.org/dna/sequencer/text/1.0"; + public static final String PREFIX = "text"; + } + + public static final Name COLUMN = new BasicName(Namespace.URI, "column"); + public static final Name DATA = new BasicName(Namespace.URI, "data"); + public static final Name ROW = new BasicName(Namespace.URI, "row"); +} Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/TextSequencerLexicon.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/package-info.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/package-info.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/package-info.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,30 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +/** + * The classes that provide the ability to sequence fixed-width and delimited text files. The sequencers in this package + * treat text files as a series of EOL-delimited rows that contain text columns that are delimited sequencer-specific ways. + */ + +package org.jboss.dna.sequencer.text; + Property changes on: trunk/extensions/dna-sequencer-text/src/main/java/org/jboss/dna/sequencer/text/package-info.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/TextSequencerI18n.properties =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/TextSequencerI18n.properties (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/TextSequencerI18n.properties 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,25 @@ +# +# JBoss DNA (http://www.jboss.org/dna) +# See the COPYRIGHT.txt file distributed with this work for information +# regarding copyright ownership. Some portions may be licensed +# to Red Hat, Inc. under one or more contributor license agreements. +# See the AUTHORS.txt file in the distribution for a full listing of +# individual contributors. +# +# JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA +# is licensed to you under the terms of the GNU Lesser General Public License as +# published by the Free Software Foundation; either version 2.1 of +# the License, or (at your option) any later version. +# +# JBoss DNA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this software; if not, write to the Free +# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA, or see the FSF site: http://www.fsf.org. +# +errorReadingLine = Error reading line from resource at path "{0}" +couldNotInstantiateRowFactory = Error instantiating RowFactory implementation class "{0}" Added: trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/sequencer-text.cnd =================================================================== --- trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/sequencer-text.cnd (rev 0) +++ trunk/extensions/dna-sequencer-text/src/main/resources/org/jboss/dna/sequencer/text/sequencer-text.cnd 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,28 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ + +<text='http://www.jboss.org/dna/sequencer/text/1.0'> + +[text:column] mixin +- text:data(string) Added: trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/DelimitedTextSequencerTest.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/DelimitedTextSequencerTest.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/DelimitedTextSequencerTest.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,257 @@ +package org.jboss.dna.sequencer.text; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertThat; +import java.io.InputStream; +import org.jboss.dna.graph.JcrNtLexicon; +import org.jboss.dna.graph.property.Name; +import org.jboss.dna.graph.sequencer.MockSequencerContext; +import org.jboss.dna.graph.sequencer.MockSequencerOutput; +import org.jboss.dna.graph.sequencer.SequencerOutput; +import org.jboss.dna.graph.sequencer.StreamSequencerContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class DelimitedTextSequencerTest { + private DelimitedTextSequencer sequencer; + private InputStream input; + private MockSequencerOutput output; + private StreamSequencerContext context; + + @Before + public void beforeEach() throws Exception { + sequencer = new DelimitedTextSequencer(); + context = new MockSequencerContext(); + context.getNamespaceRegistry().register("text", "http://www.jboss.org/dna/sequencer/text/1.0"); + context.getNamespaceRegistry().register("nt", "http://www.jcp.org/jcr/nt/1.0"); + output = new MockSequencerOutput(context); + } + + @After + public void afterEach() throws Exception { + if (input != null) { + try { + input.close(); + } finally { + input = null; + } + } + } + + protected InputStream getTestDocument( String resourcePath ) { + return this.getClass().getResourceAsStream("/delimited/" + resourcePath); + } + + @Test + public void shouldSequenceCommaDelimitedFileWithOneLine() { + input = getTestDocument("oneLineCommaDelimitedFile.csv"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + @Test + public void shouldSequenceCommaDelimitedFileWithOneLineAndNoTrailingNewLine() { + input = getTestDocument("oneLineCommaDelimitedFileNoTrailingNewLine.csv"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + @Test + public void shouldSequenceCommaDelimitedFileWithMultipleLines() { + input = getTestDocument("multiLineCommaDelimitedFile.csv"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + } + + @Test + public void shouldSequenceCommaDelimitedFileWithMultipleLinesAndMissingRecords() { + input = getTestDocument("multiLineCommaDelimitedFileMissingRecords.csv"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + // This file only has one record in line 3 + if (row == 3 && col > 1) { + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType"), is(false)); + + } else { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + } + } + + @Test + public void shouldSequencePipeDelimitedFileWithMultipleLines() { + input = getTestDocument("multiLinePipeDelimitedFile.csv"); + + sequencer.setSplitPattern("\\|"); + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + } + + @Test + public void shouldSequenceCommaDelimitedFileWithCustomRowFactory() throws Exception { + input = getTestDocument("multiLineCommaDelimitedFile.csv"); + + sequencer.setRowFactoryClassName(CustomRowFactory.class.getName()); + sequencer.sequence(input, output, context); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 0; col < columns.length; col++) { + assertThat((String)output.getProperty("text:row[" + row + "]", "text:data" + col).getFirstValue(), + is(columns[col])); + } + } + } + + @Test + public void shouldSequenceCommaDelimitedFileWithComments() throws Exception { + input = getTestDocument("multiLineCommaDelimitedFileWithComments.csv"); + + sequencer.setCommentMarker("#"); + sequencer.sequence(input, output, context); + + final int ROW_COUNT = 4; + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int row = 1; row <= ROW_COUNT; row++) { + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + // There should be no row 5, as the input file contains 6 lines, but two are commented out + int row = 5; + for (int col = 1; col <= columns.length; col++) { + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType"), is(false)); + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes"), is(false)); + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data"), is(false)); + } + } + + @Test + public void shouldSequenceCommaDelimitedFileUpToMaximumLinesSetting() { + input = getTestDocument("multiLineCommaDelimitedFile.csv"); + + sequencer.setMaximumLinesToRead(3); + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + final int ROW_COUNT = 3; + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int row = 1; row <= ROW_COUNT; row++) { + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + for (int row = 4; row <= 6; row++) { + for (int col = 1; col <= columns.length; col++) { + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType"), is(false)); + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes"), is(false)); + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data"), is(false)); + } + } + + } + public static class CustomRowFactory implements RowFactory { + + int rowNum = 1; + + @Override + public void recordRow( StreamSequencerContext context, + SequencerOutput output, + String[] columns ) { + for (int i = 0; i < columns.length; i++) { + output.setProperty("text:row[" + rowNum + "]", "text:data" + i, columns[i]); + } + + rowNum++; + } + + } +} Property changes on: trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/DelimitedTextSequencerTest.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencerTest.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencerTest.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencerTest.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,202 @@ +package org.jboss.dna.sequencer.text; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertThat; +import java.io.InputStream; +import org.jboss.dna.graph.JcrNtLexicon; +import org.jboss.dna.graph.property.Name; +import org.jboss.dna.graph.sequencer.MockSequencerContext; +import org.jboss.dna.graph.sequencer.MockSequencerOutput; +import org.jboss.dna.graph.sequencer.SequencerOutput; +import org.jboss.dna.graph.sequencer.StreamSequencerContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class FixedWidthTextSequencerTest { + private FixedWidthTextSequencer sequencer; + private InputStream input; + private MockSequencerOutput output; + private StreamSequencerContext context; + + @Before + public void beforeEach() throws Exception { + sequencer = new FixedWidthTextSequencer(); + context = new MockSequencerContext(); + context.getNamespaceRegistry().register("text", "http://www.jboss.org/dna/sequencer/text/1.0"); + context.getNamespaceRegistry().register("nt", "http://www.jcp.org/jcr/nt/1.0"); + output = new MockSequencerOutput(context); + + sequencer.setColumnStartPositions("3,6"); + } + + @After + public void afterEach() throws Exception { + if (input != null) { + try { + input.close(); + } finally { + input = null; + } + } + } + + protected InputStream getTestDocument( String resourcePath ) { + return this.getClass().getResourceAsStream("/fixed/" + resourcePath); + } + @Test + public void shouldSequenceFixedWidthFileWithOneLine() { + input = getTestDocument("oneLineFixedWidthFile.txt"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + @Test + public void shouldSequenceFixedWidthFileWithOneLineAndNoTrailingNewLine() { + input = getTestDocument("oneLineFixedWidthFileNoTrailingNewLine.txt"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + @Test + public void shouldSequenceFixedWidthFileWithMultipleLines() { + input = getTestDocument("multiLineFixedWidthFile.txt"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + } + + @Test + public void shouldSequenceFixedWidthFileWithMultipleLinesAndMissingRecords() { + input = getTestDocument("multiLineFixedWidthFileMissingRecords.txt"); + + sequencer.sequence(input, output, context); + + assertThat(output.getProperty("text:row", "jcr:primaryType"), is(notNullValue())); + assertThat((Name)output.getProperty("text:row", "jcr:primaryType").getFirstValue(), is(JcrNtLexicon.UNSTRUCTURED)); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 1; col <= columns.length; col++) { + // This file only has one record in line 3 + if (row == 3 && col > 1) { + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType"), is(false)); + + } else { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + } + } + + @Test + public void shouldSequenceFixedWidthFileWithCustomRowFactory() throws Exception { + input = getTestDocument("multiLineFixedWidthFile.txt"); + + sequencer.setRowFactoryClassName(CustomRowFactory.class.getName()); + sequencer.sequence(input, output, context); + + final int ROW_COUNT = 6; + for (int row = 1; row <= ROW_COUNT; row++) { + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int col = 0; col < columns.length; col++) { + assertThat((String)output.getProperty("text:row[" + row + "]", "text:data" + col).getFirstValue(), + is(columns[col])); + } + } + } + + @Test + public void shouldSequenceFixedWidthFileWithComments() throws Exception { + input = getTestDocument("multiLineFixedWidthFileWithComments.txt"); + + sequencer.setCommentMarker("#"); + sequencer.sequence(input, output, context); + + final int ROW_COUNT = 4; + String[] columns = new String[] {"foo", "bar", "baz"}; + for (int row = 1; row <= ROW_COUNT; row++) { + for (int col = 1; col <= columns.length; col++) { + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType").getFirstValue(), + is(JcrNtLexicon.UNSTRUCTURED)); + assertThat((Name)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes").getFirstValue(), + is(TextSequencerLexicon.COLUMN)); + assertThat((String)output.getProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data").getFirstValue(), + is(columns[col - 1])); + } + } + + // There should be no row 5, as the input file contains 6 lines, but two are commented out + int row = 5; + for (int col = 1; col <= columns.length; col++) { + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:primaryType"), is(false)); + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "jcr:mixinTypes"), is(false)); + assertThat(output.hasProperty("text:row[" + row + "]/text:column[" + col + "]", "text:data"), is(false)); + } + } + + public static class CustomRowFactory implements RowFactory { + + int rowNum = 1; + + @Override + public void recordRow( StreamSequencerContext context, + SequencerOutput output, + String[] columns ) { + for (int i = 0; i < columns.length; i++) { + output.setProperty("text:row[" + rowNum + "]", "text:data" + i, columns[i]); + } + + rowNum++; + } + + } +} Property changes on: trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/FixedWidthTextSequencerTest.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/TextSequencerI18nTest.java =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/TextSequencerI18nTest.java (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/TextSequencerI18nTest.java 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,36 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.dna.sequencer.text; + +import org.jboss.dna.common.AbstractI18nTest; + +/** + * @author Randall Hauch + */ +public class TextSequencerI18nTest extends AbstractI18nTest { + + public TextSequencerI18nTest() { + super(TextSequencerI18n.class); + } +} Property changes on: trunk/extensions/dna-sequencer-text/src/test/java/org/jboss/dna/sequencer/text/TextSequencerI18nTest.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFile.csv =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFile.csv (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFile.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +foo,bar,baz +foo,bar,baz +foo,bar,baz +foo,bar,baz +foo,bar,baz +foo,bar,baz Added: trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileMissingRecords.csv =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileMissingRecords.csv (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileMissingRecords.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +foo,bar,baz +foo,bar,baz +foo +foo,bar,baz +foo,bar,baz +foo,bar,baz Added: trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileWithComments.csv =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileWithComments.csv (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLineCommaDelimitedFileWithComments.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +# This line should be ignored completely +foo,bar,baz +foo,bar,baz +foo,bar,baz +# So should this one +foo,bar,baz Added: trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLinePipeDelimitedFile.csv =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLinePipeDelimitedFile.csv (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/multiLinePipeDelimitedFile.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +foo|bar|baz +foo|bar|baz +foo|bar|baz +foo|bar|baz +foo|bar|baz +foo|bar|baz Added: trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFile.csv =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFile.csv (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFile.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1 @@ +foo,bar,baz Added: trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFileNoTrailingNewLine.csv =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFileNoTrailingNewLine.csv (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/delimited/oneLineCommaDelimitedFileNoTrailingNewLine.csv 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1 @@ +foo,bar,baz \ No newline at end of file Added: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFile.txt =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFile.txt (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFile.txt 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +foobarbaz +foobarbaz +foobarbaz +foobarbaz +foobarbaz +foobarbaz Property changes on: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFile.txt ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileMissingRecords.txt =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileMissingRecords.txt (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileMissingRecords.txt 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +foobarbaz +foobarbaz +foo +foobarbaz +foobarbaz +foobarbaz Property changes on: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileMissingRecords.txt ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileWithComments.txt =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileWithComments.txt (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileWithComments.txt 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1,6 @@ +# This line should be ignored completely +foobarbaz +foobarbaz +foobarbaz +# So should this one +foobarbaz Property changes on: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/multiLineFixedWidthFileWithComments.txt ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFile.txt =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFile.txt (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFile.txt 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1 @@ +foobarbaz Property changes on: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFile.txt ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Added: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFileNoTrailingNewLine.txt =================================================================== --- trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFileNoTrailingNewLine.txt (rev 0) +++ trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFileNoTrailingNewLine.txt 2009-12-22 21:33:04 UTC (rev 1475) @@ -0,0 +1 @@ +foobarbaz \ No newline at end of file Property changes on: trunk/extensions/dna-sequencer-text/src/test/resources/fixed/oneLineFixedWidthFileNoTrailingNewLine.txt ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:eol-style + LF Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2009-12-22 21:13:43 UTC (rev 1474) +++ trunk/pom.xml 2009-12-22 21:33:04 UTC (rev 1475) @@ -131,6 +131,7 @@ <module>extensions/dna-sequencer-cnd</module> <module>extensions/dna-sequencer-java</module> <module>extensions/dna-sequencer-msoffice</module> + <module>extensions/dna-sequencer-text</module> <module>extensions/dna-sequencer-xml</module> <module>extensions/dna-sequencer-zip</module> <module>extensions/dna-sequencer-ddl</module>