Author: manik.surtani(a)jboss.com Date: 2008-02-12 13:38:52 -0500 (Tue, 12 Feb 2008) New Revision: 5345 Added: jbosscache-lucene/trunk/TODO.txt jbosscache-lucene/trunk/pom.xml jbosscache-lucene/trunk/src/ jbosscache-lucene/trunk/src/main/ jbosscache-lucene/trunk/src/main/java/ jbosscache-lucene/trunk/src/main/java/org/ jbosscache-lucene/trunk/src/main/java/org/jboss/ jbosscache-lucene/trunk/src/main/java/org/jboss/cache/ jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/ jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/ByteArrayIO.java jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/File.java jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/JBCDirectory.java jbosscache-lucene/trunk/src/test/ jbosscache-lucene/trunk/src/test/java/ jbosscache-lucene/trunk/src/test/java/org/ jbosscache-lucene/trunk/src/test/java/org/jboss/ jbosscache-lucene/trunk/src/test/java/org/jboss/cache/ jbosscache-lucene/trunk/src/test/java/org/jboss/cache/lucene/ jbosscache-lucene/trunk/src/test/java/org/jboss/cache/lucene/ByteArrayIOTest.java Log: Added: jbosscache-lucene/trunk/TODO.txt =================================================================== --- jbosscache-lucene/trunk/TODO.txt (rev 0) +++ jbosscache-lucene/trunk/TODO.txt 2008-02-12 18:38:52 UTC (rev 5345) @@ -0,0 +1,5 @@ +1. Figure out how to instantiate, pass in a cache instance +2. Provide a subtree root in the cache, a starting point +3. Retrieve cache from srcs such as JNDI and JMX +4. Test +5. Document Added: jbosscache-lucene/trunk/pom.xml =================================================================== --- jbosscache-lucene/trunk/pom.xml (rev 0) +++ jbosscache-lucene/trunk/pom.xml 2008-02-12 18:38:52 UTC (rev 5345) @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.jboss.cache</groupId> + <artifactId>jbosscache-common-parent</artifactId> + <version>1.1-SNAPSHOT</version> + </parent> + <groupId>org.jboss.cache</groupId> + <artifactId>jbosscache-lucene-plugin</artifactId> + <version>1.0-SNAPSHOT</version> + <name>JBoss Cache Lucene integration plugin</name> + <description>JBoss Cache Lucene integration plugin</description> + <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>org.jboss.cache</groupId> + <artifactId>jbosscache-core</artifactId> + <version>2.1.0.CR3</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>2.2.0</version> + </dependency> + <!-- test dependencies --> + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <version>2.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.jboss.transaction</groupId> + <artifactId>jboss-jta</artifactId> + <version>4.2.3.SP5</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.2-beta-1</version> + <executions> + <execution> + <id>assemble</id> + <phase>install</phase> + <goals> + <goal>attached</goal> + </goals> + <configuration> + <descriptors> + <descriptor>assembly.xml</descriptor> + </descriptors> + <finalName>${artifactId}-${version}</finalName> + <outputDirectory>target/distribution</outputDirectory> + <workDirectory>target/assembly/work</workDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <!-- basic JBoss repository so that the common parent POM in jbosscache-support can be found --> + <repositories> + <repository> + <id>repository.jboss.org</id> + <url>http://repository.jboss.org/maven2</url> + </repository> + <repository> + <id>snapshots.jboss.org</id> + <url>http://snapshots.jboss.org/maven2</url> + </repository> + </repositories> + +</project> Added: jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/ByteArrayIO.java =================================================================== --- jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/ByteArrayIO.java (rev 0) +++ jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/ByteArrayIO.java 2008-02-12 18:38:52 UTC (rev 5345) @@ -0,0 +1,191 @@ +package org.jboss.cache.lucene; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.jboss.cache.Fqn; +import org.jboss.cache.Node; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class ByteArrayIO +{ + static int getChunkNumber(long positionToRead) + { + if (positionToRead < 0) return 0; + return (int) (positionToRead >>> JBCDirectory.CHUNK_SIZE_BITS); + } + + static int getChunkPointer(long pointer) + { + return (int) (JBCDirectory.CHUNK_MASK & pointer); + } + + static byte[] getChunkData(int chunkNumber, Map<Integer, Node> chunks, Node fileNode) + { + Node chunk = chunks.get(chunkNumber); + if (chunk == null) + { + chunk = fileNode.getChild(chunkNumber); + chunks.put(chunkNumber, chunk); + } + + return (byte[]) chunk.get("buf"); + } + + public static class ByteArrayIndexInput extends IndexInput + { + private Node fileNode; + private long pointer; + private File metadata; + private Map<Integer, Node> chunks = new HashMap<Integer, Node>(); + + /** + * @param node node containing chunks as children + * @param file containing metadata + */ + public ByteArrayIndexInput(Node fileNode, File metadata) + { + this.fileNode = fileNode; + this.metadata = metadata; + } + + public byte readByte() throws IOException + { + if (pointer + 1 > metadata.getSize()) throw new IOException(metadata.getName() + ": Reading past end of file"); + return getChunkData(getChunkNumber(pointer), chunks, fileNode)[getChunkPointer(pointer++)]; + } + + public void readBytes(byte[] bytes, int offset, int length) throws IOException + { + if (pointer + length > metadata.getSize()) + throw new IOException(metadata.getName() + ": Reading past end of file"); + + int toRead = length; + int bytesReadSoFar = 0; + boolean first = true; + while (toRead > 0) + { + byte[] chunkData = getChunkData(getChunkNumber(pointer), chunks, fileNode); + int startingPoint = first ? getChunkPointer(pointer) : 0; + if (first) first = false; + int bytesToRead = Math.min(toRead, chunkData.length - startingPoint); + System.arraycopy(chunkData, startingPoint, bytes, offset + bytesReadSoFar, bytesToRead); + toRead = toRead - bytesToRead; + bytesReadSoFar += bytesToRead; + pointer += bytesToRead; + } + } + + public void close() throws IOException + { + // do nothing, except reset the pointer and release resources. + chunks.clear(); + pointer = 0; + } + + public long getFilePointer() + { + return pointer; + } + + public void seek(long l) throws IOException + { + pointer = (int) l; + } + + public long length() + { + return metadata.getSize(); + } + } + + public static class ByteArrayIndexOutput extends IndexOutput + { + private Node fileNode; + private File metadata; + private long pointer; + private Map<Integer, Node> chunks = new HashMap<Integer, Node>(); + private byte[] buffer = new byte[JBCDirectory.CHUNK_SIZE]; + private int bufferpointer; + + public ByteArrayIndexOutput(Node fileNode, File metadata) + { + this.fileNode = fileNode; + this.metadata = metadata; + } + + void newChunk() throws IOException + { + flush(); + bufferpointer = 0; + buffer = new byte[JBCDirectory.CHUNK_SIZE]; + } + + public void writeByte(byte b) throws IOException + { + if (bufferpointer == buffer.length) + { + newChunk(); + } + buffer[bufferpointer++] = b; + pointer++; + metadata.setSize(metadata.getSize() + 1); + } + + public void writeBytes(byte[] bytes, int offset, int length) throws IOException + { + + int bytesWritten = 0; + + while (bytesWritten < length) + { + int spaceInBuffer = buffer.length - bufferpointer; + int bytesToWrite = Math.min(spaceInBuffer, length - bytesWritten); + System.arraycopy(bytes, offset + bytesWritten, buffer, bufferpointer, bytesToWrite); + bytesWritten += bytesToWrite; + if (bytesWritten < length) newChunk(); + } + metadata.setSize(metadata.getSize() + length); + pointer += length; + } + + public void flush() throws IOException + { + int chunkNumber = getChunkNumber(pointer); + fileNode.addChild(new Fqn(chunkNumber)).put("buf", buffer); + } + + public void close() throws IOException + { + flush(); + bufferpointer = 0; + buffer = new byte[JBCDirectory.CHUNK_SIZE]; + pointer = 0; + } + + public long getFilePointer() + { + return pointer; + } + + public void seek(long l) throws IOException + { + if (l > metadata.getSize()) throw new IOException(metadata.getName() + ": seeking past end of file!"); + // first flush + flush(); + pointer = l; + int cn = getChunkNumber(pointer); + buffer = getChunkData(cn, chunks, fileNode); + bufferpointer = getChunkPointer(pointer); + metadata.setSize(pointer); + } + + public long length() throws IOException + { + return metadata.getSize(); + } + } + +} Added: jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/File.java =================================================================== --- jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/File.java (rev 0) +++ jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/File.java 2008-02-12 18:38:52 UTC (rev 5345) @@ -0,0 +1,72 @@ +package org.jboss.cache.lucene; + +import java.io.Serializable; + +/** + * Metadata for a file. + */ +public class File implements Serializable +{ + private long lastModified; + private String name; + private int numChunks; + private long size; + + public File(String name) + { + this.name = name; + touch(); + } + + public File(long lastModified, String name, int numChunks) + { + this.lastModified = lastModified; + this.name = name; + this.numChunks = numChunks; + } + + public long getLastModified() + { + return lastModified; + } + + public void setLastModified(long lastModified) + { + this.lastModified = lastModified; + } + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public int getNumChunks() + { + return numChunks; + } + + public void setNumChunks(int numChunks) + { + this.numChunks = numChunks; + } + + public long getSize() + { + return size; + } + + public void setSize(long size) + { + this.size = size; + } + + public void touch() + { + setLastModified(System.currentTimeMillis()); + } +} Added: jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/JBCDirectory.java =================================================================== --- jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/JBCDirectory.java (rev 0) +++ jbosscache-lucene/trunk/src/main/java/org/jboss/cache/lucene/JBCDirectory.java 2008-02-12 18:38:52 UTC (rev 5345) @@ -0,0 +1,139 @@ +package org.jboss.cache.lucene; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.jboss.cache.Cache; +import org.jboss.cache.Fqn; +import org.jboss.cache.Node; + +import java.io.IOException; +import java.util.Set; + +/** + * Structure is that each Directory maps to a Node in JBoss Cache. + * <p/> + * Each node will contain some attributes, as metadata: + * <p/> + * <ul> + * <li> + * {@link Constants#TYPE}, which could have values of {@link Constants#DIRECTORY}, or {@link Constants#FILECHUNK}. + * </li> + * <li> + * {@link org.jboss.cache.lucene.Constants#CONTENTS} - payload of the node. This only applies to FILECHUNK type nodes, + * and this is a byte array. + * </li> + * </ul> + * <p/> + * Also, in the case of DIRECTORY type nodes, additional attributes will exist. These attributes are keyed on a String, + * denoting a file name, and point to a {@link File} object as value, containing metadata for each file, in the directory + * <p/> + * Some configuration parameters include <tt>-Dorg.jboss.cache.lucene.blocksize</tt>, which defaults to <tt>"16k"</tt>. + * Other supported values for this setting include "4k", "8k", "16k", "32k", "64k" and "128k". + * This denotes the size, in bytes, of file chunks and the tradeoff between size and performance that applies to file + * system block sizes applies here. + */ +public class JBCDirectory extends Directory +{ + static String CHUNK_SIZE_STR = System.getProperty("org.jboss.cache.lucene.blocksize", "16k"); + static int CHUNK_SIZE = 1024 * Integer.parseInt(CHUNK_SIZE_STR.toUpperCase().replace("K", "")); + static int CHUNK_SIZE_BITS = (int) (Math.log(CHUNK_SIZE) / Math.log(2)); + static int CHUNK_MASK = CHUNK_SIZE - 1; + + private Cache<Object, Object> cache; + private Node<Object, Object> node; + + public JBCDirectory(Node node, Cache cache) + { + this.node = node; + this.cache = cache; + } + + public JBCDirectory(Node node) + { + this.node = node; + } + + public String[] list() throws IOException + { + Set childrenNames = node.getKeys(); + return (String[]) childrenNames.toArray(new String[]{}); + } + + public boolean fileExists(String name) throws IOException + { + return node.get(name) != null; + } + + public long fileModified(String name) throws IOException + { + return getFile(name).getLastModified(); + } + + public void touchFile(String name) throws IOException + { + getFile(name).touch(); + } + + public void deleteFile(String name) throws IOException + { + removeFile(name); + } + + public void renameFile(String from, String to) throws IOException + { + File file = getFile(from); + if (file != null) + { + Node newFile = node.addChild(new Fqn(to)); + Node oldFile = node.getChild(from); + + for (Object o : oldFile.getChildrenNames()) // chunks + { + cache.move(new Fqn(oldFile.getFqn(), o), newFile.getFqn()); + } + cache.removeNode(oldFile.getFqn()); + } + file.setName(to); + file.touch(); + node.remove(from); // remove old metadata + node.put(to, file); // new metadata + } + + public long fileLength(String name) throws IOException + { + return getFile(name).getSize(); + } + + public IndexOutput createOutput(String name) throws IOException + { + if (!fileExists(name)) + { + // create it + File file = new File(name); + node.addChild(new Fqn(name)); + node.put(name, file); + } + return new ByteArrayIO.ByteArrayIndexOutput(node.getChild(name), getFile(name)); + } + + public IndexInput openInput(String name) throws IOException + { + return new ByteArrayIO.ByteArrayIndexInput(node.getChild(name), getFile(name)); + } + + public void close() throws IOException + { + } + + protected File getFile(String name) + { + return (File) node.get(name); + } + + protected void removeFile(String name) + { + File file = (File) node.remove(name); + if (file != null) node.removeChild(file.getName()); + } +} Added: jbosscache-lucene/trunk/src/test/java/org/jboss/cache/lucene/ByteArrayIOTest.java =================================================================== --- jbosscache-lucene/trunk/src/test/java/org/jboss/cache/lucene/ByteArrayIOTest.java (rev 0) +++ jbosscache-lucene/trunk/src/test/java/org/jboss/cache/lucene/ByteArrayIOTest.java 2008-02-12 18:38:52 UTC (rev 5345) @@ -0,0 +1,221 @@ +package org.jboss.cache.lucene; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.jboss.cache.Cache; +import org.jboss.cache.DefaultCacheFactory; +import org.jboss.cache.Fqn; +import org.jboss.cache.Node; +import org.testng.annotations.Test; + +import java.io.ByteArrayOutputStream; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +@Test +public class ByteArrayIOTest +{ + public void testChunkNumbering() + { + assert ByteArrayIO.getChunkNumber(-1) == 0; + assert ByteArrayIO.getChunkNumber(0) == 0; + assert ByteArrayIO.getChunkNumber(1) == 0; + assert ByteArrayIO.getChunkNumber(JBCDirectory.CHUNK_SIZE - 1) == 0; + assert ByteArrayIO.getChunkNumber(JBCDirectory.CHUNK_SIZE) == 1; + assert ByteArrayIO.getChunkNumber(JBCDirectory.CHUNK_SIZE + 1) == 1; + } + + public void testChunkPointer() + { + assert ByteArrayIO.getChunkPointer(0) == 0; + assert ByteArrayIO.getChunkPointer(1) == 1; + assert ByteArrayIO.getChunkPointer(JBCDirectory.CHUNK_SIZE - 1) == JBCDirectory.CHUNK_SIZE - 1; + assert ByteArrayIO.getChunkPointer(JBCDirectory.CHUNK_SIZE) == 0; + assert ByteArrayIO.getChunkPointer(JBCDirectory.CHUNK_SIZE + 1) == 1; + assert ByteArrayIO.getChunkPointer(2 * JBCDirectory.CHUNK_SIZE) == 0; + assert ByteArrayIO.getChunkPointer(2 * JBCDirectory.CHUNK_SIZE + 9) == 9; + } + + // now some REAL tests + + public void readChunks() throws Exception + { + JBCDirectory.CHUNK_SIZE = 64; + JBCDirectory.CHUNK_SIZE_BITS = 6; + JBCDirectory.CHUNK_MASK = 63; + + Cache cache = new DefaultCacheFactory().createCache(); + File file1 = new File(0L, "Hello.txt", 1); + cache.put("/Dir", file1.getName(), file1); + + File file2 = new File(0L, "World.txt", 1); + cache.put("/Dir", file2.getName(), file2); + + // byte array for Hello.txt + String helloText = "Hello world. This is some text."; + cache.put(new Fqn("Dir", "Hello.txt", new Integer(0)), "buf", helloText.getBytes()); + + // byte array for World.txt - shouldbe in at least 2 chunks. + String worldText = "This String should contain more than sixty four characters but less than one hundred and twenty eight."; + + byte[] buf = new byte[JBCDirectory.CHUNK_SIZE]; + System.arraycopy(worldText.getBytes(), 0, buf, 0, JBCDirectory.CHUNK_SIZE); + cache.put(new Fqn("Dir", "World.txt", new Integer(0)), "buf", buf); + String part1 = new String(buf); + buf = new byte[JBCDirectory.CHUNK_SIZE]; + System.arraycopy(worldText.getBytes(), JBCDirectory.CHUNK_SIZE, buf, 0, worldText.length() - JBCDirectory.CHUNK_SIZE); + cache.put(new Fqn("Dir", "World.txt", new Integer(1)), "buf", buf); + String part2 = new String(buf); + // make sure the generated bytes do add up! + assert worldText.equals(part1 + part2.trim()); + + file1.setSize(helloText.length()); + file2.setSize(worldText.length()); + + // now we're set up. Nice. + + JBCDirectory dir = new JBCDirectory(cache.getNode("/Dir"), cache); + + + Set s = new HashSet(); + s.add("Hello.txt"); + s.add("World.txt"); + Set other = new HashSet(Arrays.asList(dir.list())); + + // ok, file listing works. + assert other.equals(s); + + IndexInput ii = dir.openInput("Hello.txt"); + + assert ii.length() == helloText.length(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (int i = 0; i < ii.length(); i++) baos.write(ii.readByte()); + + assert new String(baos.toByteArray()).equals(helloText); + + ii = dir.openInput("World.txt"); + + assert ii.length() == worldText.length(); + + baos = new ByteArrayOutputStream(); + + for (int i = 0; i < ii.length(); i++) baos.write(ii.readByte()); + + assert new String(baos.toByteArray()).equals(worldText); + + // now with buffered reading + + ii = dir.openInput("Hello.txt"); + + assert ii.length() == helloText.length(); + + baos = new ByteArrayOutputStream(); + + long toRead = ii.length(); + while (toRead > 0) + { + buf = new byte[19]; // suitably arbitrary + int bytesRead = (int) Math.min(toRead, 19); + ii.readBytes(buf, 0, bytesRead); + toRead = toRead - bytesRead; + baos.write(buf, 0, bytesRead); + } + + assert new String(baos.toByteArray()).equals(helloText); + + ii = dir.openInput("World.txt"); + + assert ii.length() == worldText.length(); + + baos = new ByteArrayOutputStream(); + + toRead = ii.length(); + while (toRead > 0) + { + buf = new byte[19]; // suitably arbitrary + int bytesRead = (int) Math.min(toRead, 19); + ii.readBytes(buf, 0, bytesRead); + toRead = toRead - bytesRead; + baos.write(buf, 0, bytesRead); + } + + assert new String(baos.toByteArray()).equals(worldText); + + dir.removeFile("Hello.txt"); + assert !cache.getRoot().getChild("Dir").hasChild("Hello.txt"); + + dir.renameFile("World.txt", "HelloWorld.txt"); + assert !cache.getRoot().getChild("Dir").hasChild("World.txt"); + assert cache.getRoot().getChild("Dir").hasChild("HelloWorld.txt"); + + // test that contents survive a move + ii = dir.openInput("HelloWorld.txt"); + + assert ii.length() == worldText.length(); + + baos = new ByteArrayOutputStream(); + + toRead = ii.length(); + while (toRead > 0) + { + buf = new byte[19]; // suitably arbitrary + int bytesRead = (int) Math.min(toRead, 19); + ii.readBytes(buf, 0, bytesRead); + toRead = toRead - bytesRead; + baos.write(buf, 0, bytesRead); + } + + assert new String(baos.toByteArray()).equals(worldText); + + } + + public void writeChunks() throws Exception + { + JBCDirectory.CHUNK_SIZE = 64; + JBCDirectory.CHUNK_SIZE_BITS = 6; + JBCDirectory.CHUNK_MASK = 63; + + Cache cache = new DefaultCacheFactory().createCache(); + Node d = cache.getRoot().addChild(Fqn.fromString("/Dir")); + + JBCDirectory dir = new JBCDirectory(d, cache); + + IndexOutput io = dir.createOutput("MyNewFile.txt"); + + io.writeByte((byte) 66); + io.writeByte((byte) 69); + + io.close(); + + assert d.hasChild("MyNewFile.txt"); + Node f = d.getChild("MyNewFile.txt"); + assert f.hasChild(new Integer(0)); + + // test contents by reading: + byte[] buf = new byte[9]; + IndexInput ii = dir.openInput("MyNewFile.txt"); + ii.readBytes(buf, 0, (int) ii.length()); + + assert new String(new byte[]{66, 69}).equals(new String(buf).trim()); + + String testText = "This is some rubbish again that will span more than one chunk - one hopes. Who knows, maybe even three or four chunks."; + io.seek(0); + io.writeBytes(testText.getBytes(), 0, testText.length()); + io.close(); + // now compare. + assert d.hasChild("MyNewFile.txt"); + f = d.getChild("MyNewFile.txt"); + assert f.hasChild(new Integer(0)); + assert f.hasChild(new Integer(1)); + + Node c0 = f.getChild(new Integer(0)); + Node c1 = f.getChild(new Integer(1)); + + assert testText.equals(new String((byte[]) c0.get("buf")) + new String((byte[]) c1.get("buf")).trim()); + + } + +}