[infinispan-commits] Infinispan SVN: r201 - in trunk/cachestore/s3: src/main/java/org/infinispan/loader/s3 and 1 other directories.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Thu Apr 30 10:22:30 EDT 2009
Author: adriancole
Date: 2009-04-30 10:22:30 -0400 (Thu, 30 Apr 2009)
New Revision: 201
Modified:
trunk/cachestore/s3/pom.xml
trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Bucket.java
trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3CacheStore.java
trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Connection.java
trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsBucket.java
trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsConnection.java
Log:
updated to latest snapshot of jclouds-s3
Modified: trunk/cachestore/s3/pom.xml
===================================================================
--- trunk/cachestore/s3/pom.xml 2009-04-29 15:08:34 UTC (rev 200)
+++ trunk/cachestore/s3/pom.xml 2009-04-30 14:22:30 UTC (rev 201)
@@ -36,11 +36,6 @@
<dependency>
<groupId>org.jclouds</groupId>
- <artifactId>jclouds-s3</artifactId>
- <version>1.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.jclouds</groupId>
<artifactId>jclouds-s3nio</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
Modified: trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Bucket.java
===================================================================
--- trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Bucket.java 2009-04-29 15:08:34 UTC (rev 200)
+++ trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Bucket.java 2009-04-30 14:22:30 UTC (rev 201)
@@ -5,7 +5,6 @@
import java.util.Set;
/**
- * // TODO: Adrian: Document this!
* <p/>
* This interface defines the interactons between the {@link S3CacheStore} and Amazon S3.
*
@@ -14,20 +13,50 @@
*/
public interface S3Bucket<B, C extends S3Connection> {
+ /**
+ * Creates a connection to S3, and associates this object with an S3Bucket.
+ *
+ * @param connection - third-party connection to S3
+ * @param bucket - third-party representation of an S3 Bucket
+ */
void init(C connection, B bucket);
+ /**
+ * @return name of the S3Bucket data will be stored in
+ */
String getName();
+ /**
+ * Adds the Infinispan bucket into the S3 Bucket at location {@link org.infinispan.loader.bucket.Bucket#getBucketName()}
+ * @param object what to persist into S3
+ */
void insert(Bucket object) throws S3ConnectionException;
+ /**
+ * @param key - location in the S3Bucket where we can find the infinispan Bucket
+ * @return Infinispan bucket associated with the key
+ */
Bucket get(String key) throws S3ConnectionException;
+ /**
+ * @return names all infinispan buckets stored in this S3 Bucket
+ */
Set<String> keySet() throws S3ConnectionException;
+ /**
+ * @return all infinispan buckets stored in this S3 Bucket
+ */
Set<Bucket> values() throws S3ConnectionException;
+ /**
+ * Removes the Infinispan bucket from the S3 Bucket at location <code>key</code>
+ * @param key what to remove from S3
+ */
public void remove(String key) throws S3ConnectionException;
+ /**
+ * removes all Infinispan buckets from the S3 bucket
+ */
void clear() throws S3ConnectionException;
}
\ No newline at end of file
Modified: trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3CacheStore.java
===================================================================
--- trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3CacheStore.java 2009-04-29 15:08:34 UTC (rev 200)
+++ trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3CacheStore.java 2009-04-30 14:22:30 UTC (rev 201)
@@ -17,7 +17,7 @@
import java.util.Set;
/**
- * A TODO link implementation of a {@link org.infinispan.loader.bucket.BucketBasedCacheStore}. This file store stores
+ * By default, a JClouds implementation of a {@link org.infinispan.loader.bucket.BucketBasedCacheStore}. This file store stores
* stuff in the following format: <tt>http://s3.amazon.com/{bucket}/bucket_number.bucket</tt>
* <p/>
*
Modified: trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Connection.java
===================================================================
--- trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Connection.java 2009-04-29 15:08:34 UTC (rev 200)
+++ trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/S3Connection.java 2009-04-30 14:22:30 UTC (rev 201)
@@ -3,12 +3,13 @@
import org.infinispan.marshall.Marshaller;
/**
- * // TODO: Adrian: Document this!
+ * Represents a connection to Amazon S3.
*
* @author Adrian Cole
* @since 4.0
*/
public interface S3Connection<C, B> {
+
void connect(S3CacheStoreConfig config, Marshaller m) throws S3ConnectionException;
C getConnection() throws S3ConnectionException;
Modified: trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsBucket.java
===================================================================
--- trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsBucket.java 2009-04-29 15:08:34 UTC (rev 200)
+++ trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsBucket.java 2009-04-30 14:22:30 UTC (rev 201)
@@ -4,74 +4,81 @@
import org.infinispan.loader.s3.S3ConnectionException;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
+import org.jclouds.aws.s3.S3ObjectMap;
import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.util.ArrayList;
import java.util.HashSet;
-import java.util.List;
+import java.util.Map;
import java.util.Set;
-import java.util.concurrent.Future;
/**
- * A {@link org.jclouds.aws.s3.S3Connection jclouds} implementation of {@link org.infinispan.loader.s3.S3Bucket}.
+ * A {@link org.jclouds.aws.s3.S3Connection JClouds} implementation of {@link org.infinispan.loader.s3.S3Bucket}.
* <p/>
- * Tuning and configuration parameters can be overridden by creating <tt>jets3t.properties</tt> and adding it to your
+ * Tuning and configuration parameters can be overridden by creating <tt>jclouds.properties</tt> and adding it to your
* classpath.
*
* @author Adrian Cole
- * @link http://jets3t.s3.amazonaws.com/toolkit/configuration.html
- * @since 1.0
+ * @link http://code.google.com/p/jclouds
+ * @since 4.0
*/
public class JCloudsBucket implements org.infinispan.loader.s3.S3Bucket<S3Bucket, org.infinispan.loader.s3.jclouds.JCloudsConnection> {
+
+ private JCloudsConnection connection;
private String name;
+ private S3Bucket rootS3Bucket;
+ private Map<String, InputStream> map;
public void init(JCloudsConnection connection, S3Bucket bucket) {
this.connection = connection;
this.rootS3Bucket = bucket;
this.name = bucket.getName();
+ this.map = connection.context.createMapView(rootS3Bucket);
}
public String getName() {
return name;
}
- private JCloudsConnection connection;
- protected S3Bucket rootS3Bucket;
-
/**
* {@inheritDoc}
*/
public Bucket get(String key) throws S3ConnectionException {
- Bucket bucket = null;
- InputStream is = null;
- ObjectInputStream ois = null;
-
+ InputStream input = null;
try {
- S3Object s3Object = connection.getConnection().getObject(rootS3Bucket, key).get();
+ input = map.get(key);
// it is possible that the object never existed. in this case, fall out.
- if (s3Object != null && s3Object.getContent() != null) {
- is = (InputStream) s3Object.getContent();
- bucket = (Bucket) connection.marshaller.objectFromStream(is);
- // TODO hack until we are sure the bucket has an immutable name
- bucket.setBucketName(key);
+ if (input != null) {
+ return bucketFromStream(key, input);
}
+ return null;
} catch (Exception e) {
throw connection.convertToS3ConnectionException("Error while reading from object: " + key, e);
} finally {
- safeClose(ois);
- safeClose(is);
+ safeClose(input);
}
- return bucket;
}
+ private Bucket bucketFromStream(String key, InputStream input) throws S3ConnectionException {
+ try {
+ Bucket bucket = (Bucket) connection.marshaller.objectFromStream(input);
+ // TODO hack until we are sure the bucket has an immutable name
+ bucket.setBucketName(key);
+ return bucket;
+ } catch (Exception e) {
+ throw connection.convertToS3ConnectionException("Error while reading from object: " + key, e);
+ } finally {
+ safeClose(input);
+ }
+ }
+
/**
* {@inheritDoc}
*/
+
public void remove(String key) throws S3ConnectionException {
try {
- connection.getConnection().deleteObject(rootS3Bucket, key).get();
+ map.remove(key);
} catch (Exception ex) {
throw connection.convertToS3ConnectionException("Exception removing key " + key, ex);
}
@@ -86,10 +93,8 @@
public Set<Bucket> values() throws S3ConnectionException {
Set<Bucket> buckets = new HashSet<Bucket>();
- for (String s : keySet()) {
- Bucket bucket = get(s);
- if (bucket != null)
- buckets.add(bucket);
+ for (Map.Entry<String, InputStream> entry : map.entrySet()) {
+ buckets.add(bucketFromStream(entry.getKey(), entry.getValue()));
}
return buckets;
}
@@ -99,14 +104,7 @@
*/
public void clear() throws S3ConnectionException {
try {
- List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
- for (String key : keySet()) {
- deletes.add(connection.getConnection().deleteObject(rootS3Bucket, key));
- }
- for (Future<Boolean> delete : deletes) {
- if (!delete.get())
- throw new S3ConnectionException("could not delete entry");
- }
+ map.clear();
} catch (Exception ex) {
throw connection.convertToS3ConnectionException("Exception clearing store", ex);
}
@@ -118,15 +116,9 @@
public void insert(Bucket b) throws S3ConnectionException {
try {
if (b.getEntries().isEmpty()) {
- if (!connection.getConnection().deleteObject(rootS3Bucket, b.getBucketName()).get())
- throw new S3ConnectionException(String.format("Could not delete object [%2s] in s3bucket [%1s] ", rootS3Bucket.getName(), b.getBucketName()));
+ map.remove(b.getBucketName());
} else {
- S3Object s3Object = new S3Object();
- s3Object.setKey(b.getBucketName());
- s3Object.setContent(connection.marshaller.objectToByteBuffer(b));
- s3Object.setContentType("application/octet-string");
- String id = connection.getConnection().addObject(rootS3Bucket, s3Object).get();
- assert id != null : String.format("Should have received an id for entry %1s:%2s ", rootS3Bucket.getName(), b.getBucketName());
+ ((S3ObjectMap)map).putBytes(b.getBucketName(),connection.marshaller.objectToByteBuffer(b));
}
} catch (Exception ex) {
throw connection.convertToS3ConnectionException("Exception while saving bucket " + b, ex);
Modified: trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsConnection.java
===================================================================
--- trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsConnection.java 2009-04-29 15:08:34 UTC (rev 200)
+++ trunk/cachestore/s3/src/main/java/org/infinispan/loader/s3/jclouds/JCloudsConnection.java 2009-04-30 14:22:30 UTC (rev 201)
@@ -4,46 +4,50 @@
import org.infinispan.loader.s3.S3Connection;
import org.infinispan.loader.s3.S3ConnectionException;
import org.infinispan.marshall.Marshaller;
-import org.jclouds.aws.s3.S3ConnectionFactory;
-import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
+import org.jclouds.aws.s3.S3Constants;
+import org.jclouds.aws.s3.S3Context;
+import org.jclouds.aws.s3.S3ContextFactory;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
+import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
-import java.io.IOException;
import java.io.InputStream;
-import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
/**
- * // TODO: Adrian: Document this!
+ * An JClouds implementation of {@link S3Connection}. This implementation uses the threadsafe {@link S3HttpNioConnectionPoolClientModule} transport.
*
* @author Adrian Cole
+ * @link http://code.google.com/p/jclouds
*/
public class JCloudsConnection implements S3Connection<org.jclouds.aws.s3.S3Connection, org.jclouds.aws.s3.domain.S3Bucket> {
protected org.jclouds.aws.s3.S3Connection s3Service;
+ protected S3Context context;
protected Marshaller marshaller;
/**
* {@inheritDoc}
*/
public void connect(S3CacheStoreConfig config, Marshaller m) throws S3ConnectionException {
- //TODO max connections
- //TODO proxy host/port
InputStream propertiesIS = null;
try {
propertiesIS = JCloudsConnection.class.getResourceAsStream("/jclouds.properties");
Properties properties = new Properties();
properties.load(propertiesIS);
if (!config.isSecure()) {
- properties.put("jclouds.http.port", "80");
- properties.put("jclouds.http.secure", "false");
+ properties.put(S3Constants.PROPERTY_HTTP_PORT, "80");
+ properties.put(S3Constants.PROPERTY_HTTP_SECURE, "false");
}
- if (!properties.containsKey("jclouds.aws.accesskeyid"))
- properties.put("jclouds.aws.accesskeyid", config.getAwsAccessKey());
- if (!properties.containsKey("jclouds.aws.secretaccesskey"))
- properties.put("jclouds.aws.secretaccesskey", config.getAwsSecretKey());
- this.s3Service = S3ConnectionFactory.getConnection(properties, new S3HttpNioConnectionPoolClientModule());
+ if (!properties.containsKey(S3Constants.PROPERTY_AWS_ACCESSKEYID))
+ properties.put(S3Constants.PROPERTY_AWS_ACCESSKEYID, config.getAwsAccessKey());
+ if (!properties.containsKey(S3Constants.PROPERTY_AWS_SECRETACCESSKEY))
+ properties.put(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, config.getAwsSecretKey());
+ if (!properties.containsKey(S3Constants.PROPERTY_POOL_MAX_CONNECTIONS))
+ properties.put(S3Constants.PROPERTY_POOL_MAX_CONNECTIONS, config.getMaxConnections());
+ //TODO proxy host/port
+ this.context = S3ContextFactory.createS3Context(properties, new S3HttpNioConnectionPoolClientModule());
+ this.s3Service = context.getConnection();
if (this.s3Service == null) {
throw new S3ConnectionException("Could not connect");
}
@@ -59,7 +63,7 @@
}
/**
- * @see //TODO org.jets3t.service.S3Service#getOrCreateBucket(String)
+ * @see
*/
public org.jclouds.aws.s3.domain.S3Bucket verifyOrCreateBucket(String bucketName) throws S3ConnectionException {
try {
@@ -76,9 +80,7 @@
try {
org.jclouds.aws.s3.domain.S3Bucket bucket = new org.jclouds.aws.s3.domain.S3Bucket();
bucket.setName(name);
- for (S3Object object : s3Service.getBucket(bucket).get().getContents()) {
- s3Service.deleteObject(bucket, object.getKey());
- }
+ context.createMapView(bucket).clear();
s3Service.deleteBucket(bucket);
} catch (Exception ex) {
throw convertToS3ConnectionException("Exception removing s3 bucket " + name, ex);
@@ -87,23 +89,13 @@
Set<String> keysInBucket(S3Bucket bucket) throws S3ConnectionException {
- try {
- Set<String> keys = new HashSet<String>();
- for (S3Object object : s3Service.getBucket(bucket).get().getContents()) {
- keys.add(object.getKey());
- }
- return keys;
- } catch (Exception ex) {
- throw convertToS3ConnectionException("Exception while listing bucket " + bucket, ex);
- }
+ return context.createMapView(bucket).keySet();
}
/**
* {@inheritDoc}
*/
public void copyBucket(String sourceBucket, String destinationBucket) throws S3ConnectionException {
-
-
Set<String> sourceKeys = null;
try {
S3Bucket source = new S3Bucket();
@@ -128,13 +120,11 @@
}
-
+ /**
+ * @see S3Context#close
+ */
public void disconnect() {
- try {
- s3Service.close();
- } catch (IOException e) {
- e.printStackTrace(); // TODO: Adrian: Customise this generated block
- }
+ context.close();
}
S3ConnectionException convertToS3ConnectionException(String message, Exception caught) {
More information about the infinispan-commits
mailing list