[jbosscache-commits] JBoss Cache SVN: r6969 - in core/branches/flat/src/main/java/org/jboss/starobrno: lock and 1 other directory.

Author: jason.greene(a)jboss.com Date: 2008-10-15 21:00:50 -0400 (Wed, 15 Oct 2008) New Revision: 6969 Added: core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoader.java core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoaderConfig.java core/branches/flat/src/main/java/org/jboss/starobrno/lock/StripedLock.java Modified: core/branches/flat/src/main/java/org/jboss/starobrno/loader/CacheLoader.java Log: Port FileCacheLoader Modified: core/branches/flat/src/main/java/org/jboss/starobrno/loader/CacheLoader.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/loader/CacheLoader.java 2008-10-16 00:50:31 UTC (rev 6968) +++ core/branches/flat/src/main/java/org/jboss/starobrno/loader/CacheLoader.java 2008-10-16 01:00:50 UTC (rev 6969) @@ -118,13 +118,6 @@ V put(Object key, V value) ; /** - * Removes the given key and value from the attributes of the given node. - * Does nothing if the node doesn't exist - * Returns the removed value. - */ - V remove(Object key, V value); - - /** * Removes everything from this cache-loader */ void clear(); @@ -134,7 +127,7 @@ * * @param fqn the {@link Fqn} of the node */ - void remove(Object key); + V remove(Object key); /** * Retrieves all entries stored in this cache loader Added: core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoader.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoader.java (rev 0) +++ core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoader.java 2008-10-16 01:00:50 UTC (rev 6969) @@ -0,0 +1,495 @@ +/* + * JBoss, Home of Professional Open Source. + * Copyright 2000 - 2008, Red Hat Middleware LLC, and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.starobrno.loader; + +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import net.jcip.annotations.ThreadSafe; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.jboss.starobrno.CacheException; +import org.jboss.starobrno.config.CacheLoaderConfig.IndividualCacheLoaderConfig; +import org.jboss.starobrno.lock.StripedLock; +import org.jboss.starobrno.marshall.EntryData; +import org.jboss.util.stream.MarshalledValueInputStream; + +/** + * Simple file-based CacheLoader implementation. Nodes are directories, attributes of a node is a file in the directory + * <p/> + * The FileCacheLoader has some severe limitations which restrict its use in a production + * environment, or if used in such an environment, it should be used with due care and sufficient + * understanding of these limitations. + * <ul> + * <li>Due to the way the FileCacheLoader represents a tree structure on disk (directories and files) traversal is inefficient for deep trees.</li> + * <li>Usage on shared filesystems like NFS, Windows shares, etc. should be avoided as these do not implement proper file locking and can cause data corruption.</li> + * <li>Usage with an isolation level of NONE can cause corrupt writes as multiple threads attempt to write to the same file.</li> + * <li>File systems are inherently not transactional, so when attempting to use your cache in a transactional context, failures when writing to the file (which happens during the commit phase) cannot be recovered.</li> + * </ul> + * <p/> + * As a rule of thumb, it is recommended that the FileCacheLoader not be used in a highly concurrent, + * transactional or stressful environment, and its use is restricted to testing. + * <p/> + * In terms of concurrency, file systems are notoriously inconsistent in their implementations of concurrent locks. To get around + * this and to meet the <b>thread safety</b> contracts set out in {@link CacheLoader}, this implementation uses a {@link org.jboss.cache.lock.StripedLock} + * + * @author Bela Ban + * @author <a href="mailto:galder.zamarreno@jboss.com">Galder Zamarreno</a> + * @author <a href="mailto:manik@jboss.org">Manik Surtani</a> + * @version $Id: FileCacheLoader.java 6885 2008-10-08 16:28:04Z manik.surtani(a)jboss.com $ + */ +@ThreadSafe +public class FileCacheLoader<K,V> extends AbstractCacheLoader<K,V> +{ + File root = null; + String rootPath = null; + Log log = LogFactory.getLog(getClass()); + + protected final StripedLock lock = new StripedLock(); + + private FileCacheLoaderConfig config; + + /** + * For full path, check '*' '<' '>' '|' '"' '?' Regex: [\*<>|"?] + */ + public static final Pattern PATH_PATTERN = Pattern.compile("[\\*<>|\"?]"); + + /** + * For fqn, check '*' '<' '>' '|' '"' '?' and also '\' '/' and ':' + */ + public static final Pattern KEY_PATTERN = Pattern.compile("[\\\\\\/:*<>|\"?]"); + private static boolean isOldWindows; + + static + { + float osVersion = -1; + try + { + osVersion = Float.parseFloat(System.getProperty("os.version").trim()); + } + catch (Exception e) + { + // ignore + } + // 4.x is windows NT/2000 and 5.x is XP. + isOldWindows = System.getProperty("os.name").toLowerCase().startsWith("windows") && osVersion < 4; + } + + public void setConfig(IndividualCacheLoaderConfig base) + { + if (base instanceof FileCacheLoaderConfig) + { + this.config = (FileCacheLoaderConfig) base; + } + else if (base != null) + { + this.config = new FileCacheLoaderConfig(base); + } + + String location = this.config != null ? this.config.getLocation() : null; + if (location != null && location.length() > 0) + { + root = new File(location); + rootPath = root.getAbsolutePath() + File.separator; + } + } + + public IndividualCacheLoaderConfig getConfig() + { + return config; + } + + @Override + public void create() + { + if (root == null) + { + String tmpLocation = System.getProperty("java.io.tmpdir", "C:\\tmp"); + root = new File(tmpLocation); + rootPath = root.getAbsolutePath() + File.separator; + } + if (!root.exists()) + { + if (log.isTraceEnabled()) + { + log.trace("Creating cache loader location " + root); + } + + if (config.isCheckCharacterPortability()) + { + /* Before creating the root, check whether the path is character portable. Anything that comes after is part + of the fqn which is inspected later. */ + isCharacterPortableLocation(root.getAbsolutePath()); + } + + boolean created = root.mkdirs(); + if (!created) + { + throw new CacheException("Unable to create cache loader location " + root); + } + } + + if (!root.isDirectory()) + { + throw new CacheException("Cache loader location [" + root + "] is not a directory!"); + } + } + + public V get(Object key) + { + lock(key); + try + { + try + { + return loadValue(key); + } + catch (Exception e) + { + throw new CacheException(e); + } + } + finally + { + unlock(key); + } + } + + public boolean exists(Object key) + { + lock(key); + try + { + return getFile(key, false).exists(); + } + catch (IOException e) + { + throw new CacheException(e); + } + finally + { + unlock(key); + } + } + + public V put(Object key, V value) + { + lock(key); + try + { + V retval; + try + { + retval = loadValue(key); + storeValue(key, value); + return retval; + } + catch (Exception e) + { + throw new CacheException(e); + } + } + finally + { + unlock(key); + } + } + + public V remove(Object key) + { + lock(key); + try + { + V retval; + try + { + File file = getFile(key, false); + if (! file.exists()) + return null; + + retval = loadValue(key); + file.delete(); + return retval; + } + catch (Exception e) + { + throw new CacheException(e); + } + } + finally + { + unlock(key); + } + } + + private void unlock(Object key) + { + lock.releaseLock(key.toString()); + } + + private void lock(Object key) + { + lock.acquireLock(key.toString(), true); + } + + private File getDirectory(Object key, boolean create) throws IOException + { + File f = new File(getFullPath(key)); + if (!f.exists()) + { + if (create) + { + boolean make = f.mkdirs(); + if (!make) + throw new IOException("Unable to mkdirs " + f); + } + } + return f; + } + + private String getFullPath(Object key) + { + return rootPath; + } + + private void safeClose(Closeable closeable) + { + if (closeable == null) + return; + + try + { + closeable.close(); + } + catch (IOException e) + { + } + } + + private V loadValue(Object key) throws Exception + { + File child = getFile(key, false); + if (!child.exists()) + return null; + + return loadValue(child); + } + + private V loadValue(File file) throws Exception + { + FileInputStream fileIn = null; + ObjectInputStream input = null; + + try + { + fileIn = new FileInputStream(file); + input = new MarshalledValueInputStream(fileIn); + return (V) getMarshaller().objectFromObjectStream(input); + } + catch (FileNotFoundException fnfe) + { + return null; + } + finally + { + safeClose(input); + safeClose(fileIn); + } + } + + private EntryData<K, V> loadEntry(File file) throws Exception + { + FileInputStream fileIn = null; + ObjectInputStream input = null; + + try + { + fileIn = new FileInputStream(file); + input = new MarshalledValueInputStream(fileIn); + V value = (V) getMarshaller().objectFromObjectStream(input); + K key = (K) getMarshaller().objectFromObjectStream(input); + return new EntryData<K,V>(key, value); + } + catch (FileNotFoundException fnfe) + { + return null; + } + finally + { + safeClose(input); + safeClose(fileIn); + } + } + + private File getFile(Object key, boolean create) throws IOException + { + File directory = getDirectory(key, create); + File child = new File(directory, key.toString()); + if (create && !child.exists()) + { + if (config.isCheckCharacterPortability()) + { + /* Check whether the entire file path (root + fqn + data file name), is length portable */ + isLengthPortablePath(child.getAbsolutePath()); + /* Check whether the fqn tree we're trying to store could contain non portable characters */ + isCharacterPortableKey(key); + } + + if (!child.createNewFile()) + { + throw new IOException("Unable to create file: " + child); + } + } + + return child; + } + + protected void storeValue(Object key, Object value) throws Exception + { + File child = getFile(key, true); + + FileOutputStream fileOut = null; + ObjectOutputStream output = null; + try + { + fileOut = new FileOutputStream(child); + output = new ObjectOutputStream(fileOut); + getMarshaller().objectToObjectStream(value, output); + getMarshaller().objectToObjectStream(key, output); // For getAllEntries + } + finally + { + safeClose(output); + safeClose(fileOut); + } + } + + protected boolean isCharacterPortableLocation(String fileAbsolutePath) + { + Matcher matcher = PATH_PATTERN.matcher(fileAbsolutePath); + if (matcher.find()) + { + log.warn("Cache loader location ( " + fileAbsolutePath + " ) contains one of these characters: '*' '<' '>' '|' '\"' '?'"); + log.warn("Directories containing these characters are illegal in some operative systems and could lead to portability issues"); + return false; + } + + return true; + } + + protected boolean isCharacterPortableKey(Object key) + { + // getFullPath converts Object to String via toString(), so we do too + Matcher matcher = KEY_PATTERN.matcher(key.toString()); + if (matcher.find()) + { + log.warn("The key.toString() contains one of these characters: '*' '<' '>' '|' '\"' '?' '\\' '/' ':' "); + log.warn("Directories containing these characters are illegal in some operating systems and could lead to portability issues"); + return false; + } + + return true; + } + + protected boolean isLengthPortablePath(String absoluteFqnPath) + { + + if (isOldWindows && absoluteFqnPath.length() > 255) + { + log.warn("The full absolute path to the fqn that you are trying to store is bigger than 255 characters, this could lead to problems on certain Windows systems: " + absoluteFqnPath); + return false; + } + + return true; + } + + public void clear() + { + File directory = new File(rootPath); + if (! directory.exists()) + return; + + File[] files = directory.listFiles(); + if (files != null) + { + for (File file : files) + { + try + { + lock(file.getName()); + if (file.exists()) + file.delete(); + } + finally + { + unlock(file.getName()); + } + } + } + } + + public List<EntryData<K, V>> getAllEntries() + { + List<EntryData<K,V>> entries = new LinkedList<EntryData<K,V>>(); + File directory = new File(rootPath); + if (! directory.exists()) + return entries; + + File[] files = directory.listFiles(); + if (files != null) + { + for (File file : files) + { + try + { + lock(file.getName()); + if (file.exists()) + { + EntryData<K,V> entry = loadEntry(file); + if (entry != null) + entries.add(entry); + } + } + catch (Exception e) + { + } + finally + { + unlock(file.getName()); + } + } + } + + return entries; + } +} Property changes on: core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoader.java ___________________________________________________________________ Name: svn:executable + * Added: core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoaderConfig.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoaderConfig.java (rev 0) +++ core/branches/flat/src/main/java/org/jboss/starobrno/loader/FileCacheLoaderConfig.java 2008-10-16 01:00:50 UTC (rev 6969) @@ -0,0 +1,111 @@ +/* + * JBoss, Home of Professional Open Source. + * Copyright 2000 - 2008, Red Hat Middleware LLC, and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.starobrno.loader; + +import org.jboss.cache.util.Util; +import org.jboss.starobrno.config.CacheLoaderConfig.IndividualCacheLoaderConfig; +import org.jboss.starobrno.config.Dynamic; + +import java.util.Properties; + +public class FileCacheLoaderConfig extends IndividualCacheLoaderConfig +{ + private static final long serialVersionUID = 4626734068542420865L; + + private String location; + @Dynamic + private boolean checkCharacterPortability = true; + + public FileCacheLoaderConfig() + { + setClassName(FileCacheLoader.class.getName()); + } + + /** + * For use by {@link FileCacheLoader}. + * + * @param base generic config object created by XML parsing. + */ + FileCacheLoaderConfig(IndividualCacheLoaderConfig base) + { + setClassName(FileCacheLoader.class.getName()); + populateFromBaseConfig(base); + } + + public String getLocation() + { + return location; + } + + public void setLocation(String location) + { + testImmutability("location"); + this.location = location; + } + + public boolean isCheckCharacterPortability() + { + return checkCharacterPortability; + } + + public void setCheckCharacterPortability(boolean checkCharacterPortability) + { + testImmutability("check.character.portability"); + this.checkCharacterPortability = checkCharacterPortability; + } + + @Override + public void setProperties(Properties props) + { + super.setProperties(props); + + if (props != null) + { + setLocation(props.getProperty("location")); + String prop = props.getProperty("check.character.portability"); + setCheckCharacterPortability((prop == null || Boolean.valueOf(prop))); + } + } + + @Override + public boolean equals(Object obj) + { + if (obj instanceof FileCacheLoaderConfig && equalsExcludingProperties(obj)) + { + return Util.safeEquals(location, ((FileCacheLoaderConfig) obj).location); + } + return false; + } + + @Override + public int hashCode() + { + return 31 * hashCodeExcludingProperties() + (location == null ? 0 : location.hashCode()); + } + + @Override + public FileCacheLoaderConfig clone() throws CloneNotSupportedException + { + return (FileCacheLoaderConfig) super.clone(); + } + +} \ No newline at end of file Added: core/branches/flat/src/main/java/org/jboss/starobrno/lock/StripedLock.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/lock/StripedLock.java (rev 0) +++ core/branches/flat/src/main/java/org/jboss/starobrno/lock/StripedLock.java 2008-10-16 01:00:50 UTC (rev 6969) @@ -0,0 +1,168 @@ +/* + * JBoss, Home of Professional Open Source. + * Copyright 2000 - 2008, Red Hat Middleware LLC, and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.starobrno.lock; + +import net.jcip.annotations.ThreadSafe; +import org.jboss.cache.Fqn; + +import java.util.List; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * A simple implementation of lock striping, using Fqns as the keys to lock on, primarily used to help make + * {@link org.jboss.cache.loader.CacheLoader} implemtations thread safe. + * <p/> + * Backed by a set of {@link java.util.concurrent.locks.ReentrantReadWriteLock} instances, and using the {@link org.jboss.cache.Fqn} + * hashcodes to determine buckets. + * <p/> + * Since buckets are used, it doesn't matter that the Fqn in question is not removed from the lock map when no longer in + * use, since the Fqn is not referenced in this class. Rather, the hash code is used. + * <p/> + * + * @author <a href="mailto:manik@jboss.org">Manik Surtani</a> + * @since 2.0.0 + */ +@ThreadSafe +public class StripedLock +{ + private static final int DEFAULT_CONCURRENCY = 20; + private final int lockSegmentMask; + private final int lockSegmentShift; + + final ReentrantReadWriteLock[] sharedLocks; + + /** + * This constructor just calls {@link #StripedLock(int)} with a default concurrency value of 20. + */ + public StripedLock() + { + this(DEFAULT_CONCURRENCY); + } + + /** + * Creates a new StripedLock which uses a certain number of shared locks across all elements that need to be locked. + * + * @param concurrency number of threads expected to use this class concurrently. + */ + public StripedLock(int concurrency) + { + int tempLockSegShift = 0; + int numLocks = 1; + while (numLocks < concurrency) + { + ++tempLockSegShift; + numLocks <<= 1; + } + lockSegmentShift = 32 - tempLockSegShift; + lockSegmentMask = numLocks - 1; + + sharedLocks = new ReentrantReadWriteLock[numLocks]; + + for (int i = 0; i < numLocks; i++) sharedLocks[i] = new ReentrantReadWriteLock(); + } + + /** + * Blocks until a lock is acquired. + * + * @param fqn the Fqn to lock on + * @param exclusive if true, a write (exclusive) lock is attempted, otherwise a read (shared) lock is used. + */ + public void acquireLock(Object key, boolean exclusive) + { + ReentrantReadWriteLock lock = getLock(key); + + if (exclusive) + { + lock.writeLock().lock(); + } + else + { + lock.readLock().lock(); + } + } + + /** + * Releases a lock the caller may be holding. This method is idempotent. + * + * @param fqn the Fqn to release + */ + public void releaseLock(Object key) + { + ReentrantReadWriteLock lock = getLock(key); + if (lock.isWriteLockedByCurrentThread()) + { + lock.writeLock().unlock(); + } + else + { + lock.readLock().unlock(); + } + } + + final ReentrantReadWriteLock getLock(Object o) + { + return sharedLocks[hashToIndex(o)]; + } + + final int hashToIndex(Object o) + { + return (hash(o) >>> lockSegmentShift) & lockSegmentMask; + } + + /** + * Returns a hash code for non-null Object x. + * Uses the same hash code spreader as most other java.util hash tables, except that this uses the string representation + * of the object passed in. + * + * @param x the object serving as a key + * @return the hash code + */ + final int hash(Object x) + { + int h = x.hashCode(); + h ^= (h >>> 20) ^ (h >>> 12); + return h ^ (h >>> 7) ^ (h >>> 4); + } + + /** + * Releases locks on all fqns passed in. Makes multiple calls to {@link #releaseLock(org.jboss.cache.Fqn)}. This method is idempotent. + * + * @param fqns list of fqns + * @see #releaseLock(org.jboss.cache.Fqn) + */ + public void releaseAllLocks(List<Object> keys) + { + for (Object k : keys) releaseLock(k); + } + + /** + * Acquires locks on all fqns passed in. Makes multiple calls to {@link #acquireLock(org.jboss.cache.Fqn,boolean)} + * + * @param fqns list of fqns + * @param exclusive whether locks are exclusive. + * @see #acquireLock(org.jboss.cache.Fqn,boolean) + */ + public void acquireAllLocks(List<Object> keys, boolean exclusive) + { + for (Object k : keys) acquireLock(k, exclusive); + } +}