Author: genman Date: 2008-02-07 17:57:14 -0500 (Thu, 07 Feb 2008) New Revision: 5331 Added: amazon-s3/trunk/src/main/java/com/amazon/s3/GetStreamResponse.java amazon-s3/trunk/src/test/resources/ amazon-s3/trunk/src/test/resources/log4j.xml Removed: amazon-s3/trunk/src/main/java/com/amazon/thirdparty/ Modified: amazon-s3/trunk/.classpath amazon-s3/trunk/pom.xml amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java Log: Use Apache HTTP client library instead of Java URL Connection. It's more efficient (uses the same Socket) and supports more options, like "HTTP expect" exchanges Modified: amazon-s3/trunk/.classpath =================================================================== --- amazon-s3/trunk/.classpath 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/.classpath 2008-02-07 22:57:14 UTC (rev 5331) @@ -2,6 +2,7 @@ <classpath> <classpathentry kind="src" path="src/main/java"/> <classpathentry kind="src" path="src/test/java"/> + <classpathentry kind="src" path="src/test/resources"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/> <classpathentry kind="output" path="bin"/> Modified: amazon-s3/trunk/pom.xml =================================================================== --- amazon-s3/trunk/pom.xml 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/pom.xml 2008-02-07 22:57:14 UTC (rev 5331) @@ -1,13 +1,9 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> +<?xml version="1.0" encoding="UTF-8"?><project> <modelVersion>4.0.0</modelVersion> <groupId>com.amazon</groupId> <artifactId>amazon-s3-rest</artifactId> <version>1.0.0.0-SNAPSHOT</version> - <description> - Cleaned-up version of the Amazon S3 REST access layer - </description> + <description>Cleaned-up version of the Amazon S3 REST access layer</description> <build> <plugins> <plugin> @@ -18,7 +14,6 @@ </configuration> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> <systemProperties> @@ -42,5 +37,21 @@ <version>4.1</version> <scope>test</scope> </dependency> + <dependency> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + <version>3.1</version> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.14</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + <version>3.0.1</version> + </dependency> </dependencies> -</project> +</project> \ No newline at end of file Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -92,7 +92,7 @@ * Returns true if the bucket name is valid for the calling format. */ public static boolean validateBucketName(String bucketName, CallingFormat callingFormat) { - if (callingFormat == CallingFormat.getPathCallingFormat()) + if (callingFormat == CallingFormat.PATH) { final int MIN_BUCKET_LENGTH = 3; final int MAX_BUCKET_LENGTH = 255; Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -9,10 +9,13 @@ package com.amazon.s3; -import java.net.MalformedURLException; -import java.net.URL; import java.util.Map; +import org.apache.commons.httpclient.HttpURL; +import org.apache.commons.httpclient.HttpsURL; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; + /** * Calling formats. * @@ -20,9 +23,9 @@ */ public abstract class CallingFormat { - private final static CallingFormat pathCallingFormat = new PathCallingFormat(); - private final static CallingFormat subdomainCallingFormat = new SubdomainCallingFormat(); - private final static CallingFormat vanityCallingFormat = new VanityCallingFormat(); + public final static CallingFormat PATH = new PathCallingFormat(); + public final static CallingFormat SUBDOMAIN = new SubdomainCallingFormat(); + public final static CallingFormat VANITY = new VanityCallingFormat(); public abstract boolean supportsLocatedBuckets(); @@ -30,71 +33,57 @@ public abstract String getPathBase(Bucket bucket, String key); - public abstract URL getURL(boolean isSecure, String server, int port, + public abstract URI getURI(boolean isSecure, String server, int port, Bucket bucket, String key, Map<String, String> pathArgs) - throws MalformedURLException; + throws URIException; /** - * Converts the Path Arguments from a map to String which can be used in url construction - * @param pathArgs a map of arguments - * @return a string representation of pathArgs + * Adds query parameters to the URL. */ - public static String convertPathArgsHashToString(Map<String, String> pathArgs) { - StringBuilder pathArgsString = new StringBuilder(); - String argumentValue; - boolean firstRun = true; - if (pathArgs != null) { - for (Map.Entry<String, String> me : pathArgs.entrySet()) { - String argument = me.getKey(); - if (firstRun) { - firstRun = false; - pathArgsString.append("?"); - } else { - pathArgsString.append("&"); - } - - argumentValue = me.getValue(); - pathArgsString.append(argument); - if (argumentValue != null) { - pathArgsString.append("="); - pathArgsString.append(argumentValue); - } - } + private static URI addQuery(HttpURL uri, Map<String, String> pathArgs) throws URIException { + if (pathArgs == null || pathArgs.isEmpty()) + return uri; + int size = pathArgs.size(); + String[] name = new String[size]; + String[] value = new String[size]; + int i = 0; + for (Map.Entry<String, String> me : pathArgs.entrySet()) { + name[i] = me.getKey(); + value[i] = me.getValue(); + if (value[i] == null) + throw new NullPointerException("query cannot contain null " + pathArgs); + i++; } - - return pathArgsString.toString(); + uri.setQuery(name, value); + return uri; } - public static CallingFormat getPathCallingFormat() { - return pathCallingFormat; + private static URI url(boolean isSecure, String host, int port, String path, Map<String, String> pathArgs) + throws URIException + { + HttpURL url = isSecure ? new HttpsURL(host, port, "/") : new HttpURL(host, port, "/"); + url.setEscapedPath(path); + return addQuery(url, pathArgs); } - public static CallingFormat getSubdomainCallingFormat() { - return subdomainCallingFormat; - } - - public static CallingFormat getVanityCallingFormat() { - return vanityCallingFormat; - } - static private class PathCallingFormat extends CallingFormat { public boolean supportsLocatedBuckets() { return false; } public String getPathBase(Bucket bucket, String key) { - return isBucketSpecified(bucket) ? "/" + bucket + "/" + key : "/"; + return isBucketSpecified(bucket) ? "/" + bucket + "/" + UrlEncoder.encode(key) : "/"; } public String getEndpoint(String server, int port, Bucket bucket) { return server + ":" + port; } - public URL getURL(boolean isSecure, String server, int port, Bucket bucket, String key, Map<String, String> pathArgs) - throws MalformedURLException { - String pathBase = isBucketSpecified(bucket) ? "/" + bucket + "/" + key : "/"; - String pathArguments = convertPathArgsHashToString(pathArgs); - return new URL(isSecure ? "https" : "http", server, port, pathBase + pathArguments); + @Override + public URI getURI(boolean isSecure, String server, int port, Bucket bucket, String key, Map<String, String> pathArgs) + throws URIException { + String pathBase = isBucketSpecified(bucket) ? "/" + bucket + "/" + UrlEncoder.encode(key) : "/"; + return url(isSecure, server, port, pathBase, pathArgs); } private boolean isBucketSpecified(Bucket bucket) { @@ -120,22 +109,19 @@ } public String getPathBase(Bucket bucket, String key) { - return "/" + key; + return "/" + UrlEncoder.encode(key); } - public URL getURL(boolean isSecure, String server, int port, + @Override + public URI getURI(boolean isSecure, String server, int port, Bucket bucket, String key, Map<String, String> pathArgs) - throws MalformedURLException { + throws URIException { if (bucket == null || !bucket.specified()) { - // The bucket is null, this is listAllBuckets request - String pathArguments = convertPathArgsHashToString(pathArgs); - return new URL(isSecure ? "https" : "http", server, port, "/" - + pathArguments); + return url(isSecure, server, port, "", pathArgs); } else { String serverToUse = getServer(server, bucket); String pathBase = getPathBase(bucket, key); - String pathArguments = convertPathArgsHashToString(pathArgs); - return new URL(isSecure ? "https" : "http", serverToUse, port, pathBase + pathArguments); + return url(isSecure, serverToUse, port, pathBase, pathArgs); } } } Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -12,7 +12,8 @@ import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import com.amazon.thirdparty.Base64; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.httpclient.util.EncodingUtil; /** * Creates canonical strings for authorization purposes by hashing @@ -109,7 +110,7 @@ // append the key (it might be an empty string) // append a slash regardless buf.append("/"); - if(key != null) { + if (key != null) { buf.append(key); } @@ -164,29 +165,25 @@ * @param data Data to sign * @param passcode Passcode to sign it with * @return Signature - * @throws NoSuchAlgorithmException If the algorithm does not exist. Unlikely - * @throws InvalidKeyException If the key is invalid. */ static String encode(Key signingKey, String canonicalString, boolean urlencode) { - // Acquire the MAC instance and initialize with the signing key. - Mac mac = null; + Mac mac; try { mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); } catch (NoSuchAlgorithmException e) { - // should not happen throw new RuntimeException("Could not find sha1 algorithm", e); } try { mac.init(signingKey); } catch (InvalidKeyException e) { - // also should not happen throw new RuntimeException("Could not initialize the MAC algorithm", e); } - // Compute the HMAC on the digest, and set it. - String b64 = Base64.encodeBytes(mac.doFinal(canonicalString.getBytes())); + byte[] b = mac.doFinal(canonicalString.getBytes()); + byte[] encode = Base64.encodeBase64(b); + String b64 = EncodingUtil.getAsciiString(encode); if (urlencode) { return UrlEncoder.encode(b64); Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -10,8 +10,6 @@ package com.amazon.s3; import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; import java.security.Key; import java.text.SimpleDateFormat; import java.util.Collections; @@ -22,6 +20,17 @@ import java.util.Map; import java.util.TimeZone; +import org.apache.commons.httpclient.HostConfiguration; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.methods.ByteArrayRequestEntity; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.commons.httpclient.methods.StringRequestEntity; +import org.apache.commons.httpclient.params.HttpMethodParams; + /** * A stateless connection to the Amazon S3 system which uses the REST API. * <p/> @@ -36,6 +45,11 @@ public static final String DEFAULT_HOST = "s3.amazonaws.com"; public static final int INSECURE_PORT = 80; public static final int SECURE_PORT = 443; + + /** + * Data larger than 1024 bytes will use expect headers. + */ + public static final int EXPECT_SIZE = 1024; private String awsAccessKeyId; private Key awsSecretAccessKey; @@ -44,6 +58,17 @@ private int port; private CallingFormat callingFormat; + private MultiThreadedHttpConnectionManager connectionManager = + new MultiThreadedHttpConnectionManager(); + private HostConfiguration config; + private HttpClient client; + + static { + String charset = URI.getDefaultProtocolCharset(); + if (!charset.equals("UTF-8")) + throw new Error("URI charset must be UTF-8: " + charset); + } + public Connection(String awsAccessKeyId, String awsSecretAccessKey) { this(awsAccessKeyId, awsSecretAccessKey, true); } @@ -61,7 +86,7 @@ public Connection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, String server, int port) { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, port, CallingFormat.getSubdomainCallingFormat()); + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, port, CallingFormat.SUBDOMAIN); } @@ -69,7 +94,7 @@ String server, CallingFormat format) { this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, isSecure ? SECURE_PORT : INSECURE_PORT, - CallingFormat.getSubdomainCallingFormat()); + CallingFormat.SUBDOMAIN); } /** @@ -92,6 +117,11 @@ this.server = server; this.port = port; this.callingFormat = format; + + config = new HostConfiguration(); + config.setHost(server, port, isSecure ? "http" : "https"); + client = new HttpClient(connectionManager); + client.setHostConfiguration(config); } /** @@ -121,14 +151,27 @@ if (!bucket.validateName(callingFormat)) throw new IllegalArgumentException("Invalid Bucket Name: "+bucket); - HttpURLConnection request = makeRequest(Method.PUT, bucket, headers); + PutMethod method = (PutMethod) makeRequest(Method.PUT, bucket, headers); if (body != null) { - request.setDoOutput(true); - request.getOutputStream().write(body.getBytes("UTF-8")); + StringRequestEntity sre = new StringRequestEntity(body, "text/xml", "UTF-8"); + method.setRequestEntity(sre); } - return new Response(request); + executeRelease(method); + return new Response(method); } + + private int execute(HttpMethod method) throws IOException { + return client.executeMethod(method); + } + + private int executeRelease(HttpMethod method) throws IOException { + try { + return client.executeMethod(method); + } finally { + method.releaseConnection(); + } + } /** * Creates a new bucket. @@ -153,8 +196,8 @@ */ public boolean exists(Bucket bucket) throws IOException { - HttpURLConnection response = makeRequest(Method.HEAD, bucket); - int httpCode = response.getResponseCode(); + HttpMethod method = makeRequest(Method.HEAD, bucket); + int httpCode = executeRelease(method); return httpCode >= 200 && httpCode < 300; } @@ -201,7 +244,7 @@ } /** - * Lists the contents of a bucket. + * Lists the contents of a bucket by prefix. */ public ListResponse list(Bucket bucket, String prefix) throws IOException { return list(bucket, prefix, null, null); @@ -224,10 +267,34 @@ throws IOException { - Map<String, String> pathArgs = Response.paramsForListOptions(prefix, marker, maxKeys, delimiter); - return new ListResponse(makeRequest(Method.GET, bucket, pathArgs, headers)); + Map<String, String> pathArgs = paramsForListOptions(prefix, marker, maxKeys, delimiter); + HttpMethod method = makeRequest(Method.GET, bucket, pathArgs, headers); + try { + execute(method); + return new ListResponse(method); + } finally { + method.releaseConnection(); + } } + private static Map<String, String> paramsForListOptions(String prefix, + String marker, Integer maxKeys, String delimiter) + { + Map<String, String> argParams = new HashMap<String, String>(); + if (prefix != null) + argParams.put("prefix", prefix); + if (marker != null) + argParams.put("marker", marker); + if (delimiter != null) + argParams.put("delimiter", delimiter); + + if (maxKeys != null) + argParams.put("max-keys", Integer.toString(maxKeys.intValue())); + + return argParams; + + } + /** * Deletes a bucket. * @param bucket The name of the bucket to delete. @@ -237,7 +304,9 @@ public Response delete(Bucket bucket, Headers headers) throws IOException { - return new Response(makeRequest(Method.DELETE, bucket, "", null, headers)); + HttpMethod method = makeRequest(Method.DELETE, bucket, "", null, headers); + executeRelease(method); + return new Response(method); } /** @@ -260,13 +329,8 @@ public Response put(Bucket bucket, String key, S3Object object, Headers headers) throws IOException { - HttpURLConnection request = - makeRequest(Method.PUT, bucket, key, null, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.getData() == null ? new byte[] {} : object.getData()); - - return new Response(request); + PutMethod request = (PutMethod) makeRequest(Method.PUT, bucket, key, null, headers, object); + return execute(request, object); } /** @@ -280,13 +344,18 @@ * Reads an object from S3. * @param bucket The name of the bucket where the object lives. * @param key The name of the key to use. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). + * @param headers HTTP headers to pass (can be null). */ public GetResponse get(Bucket bucket, String key, Headers headers) throws IOException { - return new GetResponse(makeRequest(Method.GET, bucket, key, null, headers)); + HttpMethod method = makeRequest(Method.GET, bucket, key, null, headers); + try { + execute(method); + return new GetResponse(method); + } finally { + method.releaseConnection(); + } } /** @@ -298,6 +367,39 @@ } /** + * Reads an object from S3, returning a stream to access the data. + * This is preferable when dealing with large objects. + * + * @param bucket The name of the bucket where the object lives. + * @param key The name of the key to use. + * @param headers HTTP headers to pass (can be null). + */ + public GetStreamResponse getStream(Bucket bucket, String key, Headers headers) + throws IOException + { + HttpMethod method = makeRequest(Method.GET, bucket, key, null, headers); + boolean ok = false; + try { + execute(method); + ok = true; + return new GetStreamResponse((GetMethod)method); + } finally { + if (!ok) + method.releaseConnection(); + } + } + + /** + * Reads an object from S3, returning a stream to access the data. + * This is preferable when dealing with large objects. + */ + public GetStreamResponse getStream(Bucket bucket, String key) + throws IOException + { + return getStream(bucket, key, null); + } + + /** * Returns information about an S3 object without loading it. * Check {@link Response#isOk()} or {@link Response#isNotFound()} to see if the object exists. */ @@ -309,7 +411,9 @@ * Returns information about an S3 object without loading it. */ public Response head(Bucket bucket, String key, Headers headers) throws IOException { - return new Response(makeRequest(Method.HEAD, bucket, key, null, headers)); + HttpMethod method = makeRequest(Method.HEAD, bucket, key, null, headers); + executeRelease(method); + return new Response(method); } /** @@ -322,7 +426,9 @@ public Response delete(Bucket bucket, String key, Headers headers) throws IOException { - return new Response(makeRequest(Method.DELETE, bucket, key, null, headers)); + HttpMethod method = makeRequest(Method.DELETE, bucket, key, null, headers); + executeRelease(method); + return new Response(method); } /** @@ -342,8 +448,14 @@ public GetResponse getBucketLogging(Bucket bucket, Headers headers) throws IOException { - Map<String, String> pathArgs = Collections.singletonMap("logging", null); - return new GetResponse(makeRequest(Method.GET, bucket, "", pathArgs, headers)); + Map<String, String> pathArgs = Collections.singletonMap("logging", ""); + HttpMethod method = makeRequest(Method.GET, bucket, "", pathArgs, headers); + try { + execute(method); + return new GetResponse(method); + } finally { + method.releaseConnection(); + } } /** @@ -356,13 +468,18 @@ public Response putBucketLogging(Bucket bucket, String loggingXMLDoc, Headers headers) throws IOException { - Map<String, String> pathArgs = Collections.singletonMap("logging", null); + Map<String, String> pathArgs = Collections.singletonMap("logging", ""); S3Object object = new S3Object(loggingXMLDoc.getBytes(), null); - HttpURLConnection request = makeRequest(Method.PUT, bucket, "", pathArgs, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.getData() == null ? new byte[] {} : object.getData()); - + PutMethod request = (PutMethod) makeRequest(Method.PUT, bucket, "", pathArgs, headers, object); + return execute(request, object); + } + + private Response execute(PutMethod request, S3Object object) throws IOException { + if (object.getLength() > EXPECT_SIZE) + request.getParams().setBooleanParameter(HttpMethodParams.USE_EXPECT_CONTINUE, true); + // request.setContentChunked(true); + request.setRequestEntity(new ByteArrayRequestEntity(object.getData())); + executeRelease(request); return new Response(request); } @@ -391,10 +508,14 @@ if (key == null) key = ""; - Map<String, String> pathArgs = Collections.singletonMap("acl", null); - return new GetResponse( - makeRequest(Method.GET, bucket, key, pathArgs, headers) - ); + Map<String, String> pathArgs = Collections.singletonMap("acl", ""); + HttpMethod method = makeRequest(Method.GET, bucket, key, pathArgs, headers); + try { + execute(method); + return new GetResponse(method); + } finally { + method.releaseConnection(); + } } /** @@ -421,18 +542,12 @@ public Response putACL(Bucket bucket, String key, String aclXMLDoc, Headers headers) throws IOException { - S3Object object = new S3Object(aclXMLDoc.getBytes(), null); - - Map<String, String> pathArgs = new HashMap<String, String>(); - pathArgs.put("acl", null); + S3Object object = new S3Object(aclXMLDoc); + Map<String, String> pathArgs = Collections.singletonMap("acl", ""); + PutMethod request = + (PutMethod) makeRequest(Method.PUT, bucket, key, pathArgs, headers, object); + return execute(request, object); - HttpURLConnection request = - makeRequest(Method.PUT, bucket, key, pathArgs, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.getData() == null ? new byte[] {} : object.getData()); - - return new Response(request); } /** @@ -441,9 +556,14 @@ public LocationResponse getLocation(Bucket bucket) throws IOException { - Map<String, String> pathArgs = new HashMap<String, String>(); - pathArgs.put("location", null); - return new LocationResponse(makeRequest(Method.GET, bucket, "", pathArgs, null)); + Map<String, String> pathArgs = Collections.singletonMap("location", ""); + HttpMethod method = makeRequest(Method.GET, bucket, "", pathArgs, null); + try { + execute(method); + return new LocationResponse(method); + } finally { + method.releaseConnection(); + } } @@ -455,7 +575,13 @@ public ListAllBucketsResponse listAllBuckets(Headers headers) throws IOException { - return new ListAllBucketsResponse(makeRequest(Method.GET, null, "", null, headers)); + HttpMethod method = makeRequest(Method.GET, null, "", null, headers); + try { + execute(method); + return new ListAllBucketsResponse(method); + } finally { + method.releaseConnection(); + } } /** @@ -467,7 +593,7 @@ } /** - * Make a new HttpURLConnection without passing an S3Object parameter. + * Make a new HttpMethod without passing an S3Object parameter. * Use this method for key operations that do require arguments * @param method The method to invoke * @param bucketName the bucket this request is for @@ -477,27 +603,27 @@ * @return * @throws IOException */ - private HttpURLConnection makeRequest(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) + private HttpMethod makeRequest(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) throws IOException { return makeRequest(method, bucket, key, pathArgs, headers, null); } - private HttpURLConnection makeRequest(Method method, Bucket bucket) throws IOException { + private HttpMethod makeRequest(Method method, Bucket bucket) throws IOException { return makeRequest(method, bucket, null); } - private HttpURLConnection makeRequest(Method method, Bucket bucket, Headers headers) throws IOException { + private HttpMethod makeRequest(Method method, Bucket bucket, Headers headers) throws IOException { return makeRequest(method, bucket, null, headers); } - private HttpURLConnection makeRequest(Method method, Bucket bucket, Map<String, String> pathArgs, Headers headers) throws IOException + private HttpMethod makeRequest(Method method, Bucket bucket, Map<String, String> pathArgs, Headers headers) throws IOException { return makeRequest(method, bucket, "", pathArgs, headers); } /** - * Make a new HttpURLConnection. + * Make a new HttpMethod. * @param method The HTTP method to use (GET, PUT, DELETE) * @param bucketNamePattern The bucket name this request affects * @param key The key this request is for, not encoded @@ -506,89 +632,81 @@ * headers to pass (can be null). * @param object The S3Object that is to be written (can be null). */ - private HttpURLConnection makeRequest(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers, + private HttpMethod makeRequest(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers, S3Object object) throws IOException { - String enckey = UrlEncoder.encode(key); - // build the domain based on the calling format - URL url = this.callingFormat.getURL(this.isSecure, server, this.port, bucket, enckey, pathArgs); - - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod(method.name()); - - // subdomain-style urls may encounter http redirects. - // Ensure that redirects are supported. - if (!connection.getInstanceFollowRedirects() - && callingFormat.supportsLocatedBuckets()) - throw new RuntimeException("HTTP redirect support required."); - - addHeaders(connection, headers); + HttpMethod httpMethod = method.createHttpMethod(); + + URI uri = this.callingFormat.getURI(this.isSecure, server, this.port, bucket, key, pathArgs); + + httpMethod.setURI(uri); + addHeaders(httpMethod, headers); if (object != null) - addMetadataHeaders(connection, object.getMetadata()); - addAuthHeader(connection, method, bucket, enckey, pathArgs); - - return connection; + addMetadataHeaders(httpMethod, object.getMetadata()); + addAuthHeader(httpMethod, method, bucket, key, pathArgs); + return httpMethod; } /** - * Add the given headers to the HttpURLConnection. - * @param connection The HttpURLConnection to which the headers will be added. + * Add the given headers to the HttpMethod. + * @param httpMethod The HttpMethod to which the headers will be added. * @param headers A Map of String to List of Strings representing the http * headers to pass (can be null). */ - private void addHeaders(HttpURLConnection connection, Headers headers) { - addHeaders(connection, headers, ""); + private void addHeaders(HttpMethod httpMethod, Headers headers) { + addHeaders(httpMethod, headers, ""); } /** - * Add the given metadata fields to the HttpURLConnection. - * @param connection The HttpURLConnection to which the headers will be added. + * Add the given metadata fields to the HttpMethod. + * @param httpMethod The HttpMethod to which the headers will be added. * @param metadata A Map of String to List of Strings representing the s3 * metadata for this resource. */ - private void addMetadataHeaders(HttpURLConnection connection, Headers metadata) { - addHeaders(connection, metadata, Headers.METADATA_PREFIX); + private void addMetadataHeaders(HttpMethod httpMethod, Headers metadata) { + addHeaders(httpMethod, metadata, Headers.METADATA_PREFIX); } /** - * Add the given headers to the HttpURLConnection with a prefix before the keys. - * @param connection The HttpURLConnection to which the headers will be added. + * Add the given headers to the HttpMethod with a prefix before the keys. + * @param httpMethod The HttpMethod to which the headers will be added. * @param headers A Map of String to List of Strings representing the http * headers to pass (can be null). * @param prefix The string to prepend to each key before adding it to the connection. */ - private void addHeaders(HttpURLConnection connection, Headers headers, String prefix) { + private void addHeaders(HttpMethod httpMethod, Headers headers, String prefix) { if (headers != null) { for (Map.Entry<String, List<String>> me : headers.getHeaders().entrySet()) { String key = me.getKey(); for (String value : me.getValue()) { - connection.addRequestProperty(prefix + key, value); + httpMethod.addRequestHeader(prefix + key, value); } } } } /** - * Add the appropriate Authorization header to the HttpURLConnection. - * @param connection The HttpURLConnection to which the header will be added. + * Add the appropriate Authorization header to the HttpMethod. + * @param httpMethod The HttpMethod to which the header will be added. * @param method The HTTP method to use (GET, PUT, DELETE) * @param bucket the bucket name this request is for * @param key the key this request is for (not URL encoded) * @param pathArgs path arguments which are part of this request */ - private void addAuthHeader(HttpURLConnection connection, Method method, Bucket bucket, String enckey, Map<String, String> pathArgs) { - if (connection.getRequestProperty("Date") == null) { - connection.setRequestProperty("Date", httpDate()); + private void addAuthHeader(HttpMethod httpMethod, Method method, Bucket bucket, String key, Map<String, String> pathArgs) { + if (httpMethod.getRequestHeader("Date") == null) { + httpMethod.setRequestHeader("Date", httpDate()); } - if (connection.getRequestProperty("Content-Type") == null) { - connection.setRequestProperty("Content-Type", ""); + if (httpMethod.getRequestHeader("Content-Type") == null) { + httpMethod.setRequestHeader("Content-Type", ""); } - Headers prop = new Headers(connection.getRequestProperties()); + Headers prop = new Headers(httpMethod.getRequestHeaders()); + String enckey = UrlEncoder.encode(key); String canonicalString = CanonicalString.make(method, bucket, enckey, pathArgs, prop); String encodedCanonical = CanonicalString.encode(this.awsSecretAccessKey, canonicalString, false); - connection.setRequestProperty("Authorization", + httpMethod.setRequestHeader("Authorization", "AWS " + this.awsAccessKeyId + ":" + encodedCanonical); } @@ -603,6 +721,13 @@ } /** + * Shuts down any managed or pooled connections. + */ + public void shutdown() { + connectionManager.shutdown(); + } + + /** * Returns a debug string. */ @Override Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -12,25 +12,22 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.net.HttpURLConnection; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpMethodBase; + /** - * A Response object returned from AWSAuthConnection.get(). Exposes the - * attribute object, which represents the retrieved object. + * Returned by the {@link Connection#get(Bucket, String)} methods. */ public class GetResponse extends Response { private S3Object object; - /** - * Pulls a representation of an S3Object out of the HttpURLConnection - * response. - */ - GetResponse(HttpURLConnection connection) throws IOException { - super(connection); - if (connection.getResponseCode() < 400) { - int len = connection.getContentLength(); - byte[] body = slurpInputStream(connection.getInputStream(), len); + GetResponse(HttpMethod method) throws IOException { + super(method); + if (getResponseCode() < 400) { + long len = ((HttpMethodBase)method).getResponseContentLength(); + byte[] body = toByteArray(method.getResponseBodyAsStream(), len); object = new S3Object(body, getHeaders().extractMetadata()); } } @@ -38,22 +35,21 @@ /** * Read the input stream and dump it all into a big byte array */ - private static byte[] slurpInputStream(InputStream stream, int len) throws IOException { - + private static byte[] toByteArray(InputStream stream, long len) throws IOException { final int chunkSize = 4 * 1024; byte[] buf = new byte[chunkSize]; - ByteArrayOutputStream byteStream; - if (len > 0) { - byteStream = new ByteArrayOutputStream(len); + ByteArrayOutputStream baos; + if (len != -1) { + baos = new ByteArrayOutputStream((int)len); } else { - byteStream = new ByteArrayOutputStream(chunkSize); + baos = new ByteArrayOutputStream(chunkSize); } int count; while ((count = stream.read(buf)) != -1) - byteStream.write(buf, 0, count); + baos.write(buf, 0, count); - return byteStream.toByteArray(); + return baos.toByteArray(); } /** Added: amazon-s3/trunk/src/main/java/com/amazon/s3/GetStreamResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/GetStreamResponse.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/GetStreamResponse.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -0,0 +1,67 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.httpclient.methods.GetMethod; + +/** + * A Response object returned from Connection.getStream(). + * This response, once read, must be released by calling {@link #release()}. + */ +public class GetStreamResponse extends Response { + + private GetMethod method; + + /** + * Pulls a representation of an S3Object out of the HttpURLConnection + * response. + */ + GetStreamResponse(GetMethod method) throws IOException { + super(method); + this.method = method; + } + + /** + * Returns an input stream for the content. + * Call {@link #release}) after done reading. + * @throws IOException + */ + public InputStream getInputStream() throws IOException { + InputStream body = method.getResponseBodyAsStream(); + if (body == null) + throw new IllegalStateException("body null"); + return body; + } + + /** + * Returns the content length, if known. + */ + public long getLength() { + return method.getResponseContentLength(); + } + + /** + * Releases this connection. + */ + public void release() { + method.releaseConnection(); + } + + /** + * Returns a debug string. + */ + @Override + public String toString() { + return super.toString() + " method=" + method; + } +} Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -7,6 +7,8 @@ import java.util.Map; import java.util.TreeMap; +import org.apache.commons.httpclient.Header; + /** * HTTP header wrapper. * @@ -40,6 +42,13 @@ this(new HashMap<String, List<String>>()); } + public Headers(Header[] requestHeaders) { + this(); + for (Header h : requestHeaders) { + put(h.getName(), h.getValue()); + } + } + /** * Adds a header. */ Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -10,7 +10,6 @@ package com.amazon.s3; import java.io.IOException; -import java.net.HttpURLConnection; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -18,6 +17,7 @@ import java.util.List; import java.util.TimeZone; +import org.apache.commons.httpclient.HttpMethod; import org.xml.sax.Attributes; import org.xml.sax.helpers.DefaultHandler; @@ -28,8 +28,8 @@ private List<Bucket> entries; - ListAllBucketsResponse(HttpURLConnection connection) throws IOException { - super(connection); + ListAllBucketsResponse(HttpMethod method) throws IOException { + super(method); if (isOk()) { entries = new ArrayList<Bucket>(); parse(new ListAllMyBucketsHandler()); Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -10,13 +10,13 @@ package com.amazon.s3; import java.io.IOException; -import java.net.HttpURLConnection; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.TimeZone; +import org.apache.commons.httpclient.HttpMethod; import org.xml.sax.Attributes; import org.xml.sax.helpers.DefaultHandler; @@ -43,8 +43,8 @@ private List<CommonPrefixEntry> commonPrefixEntries = null; - ListResponse(HttpURLConnection connection) throws IOException { - super(connection); + ListResponse(HttpMethod method) throws IOException { + super(method); if (isOk()) { parse(new ListBucketHandler()); } @@ -238,13 +238,13 @@ @Override public String toString() { return super.toString() + + " entries=" + entries + " name=" + name + " prefix=" + prefix + " marker=" + marker + " maxKeys=" + maxKeys + " isTruncated=" + isTruncated + " nextMarker=" + nextMarker + - " entries=" + entries + " prefix=" + commonPrefixEntries; } Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -10,8 +10,8 @@ package com.amazon.s3; import java.io.IOException; -import java.net.HttpURLConnection; +import org.apache.commons.httpclient.HttpMethod; import org.xml.sax.Attributes; import org.xml.sax.helpers.DefaultHandler; @@ -27,8 +27,8 @@ /** * Parse the response to a ?location query. */ - LocationResponse(HttpURLConnection connection) throws IOException { - super(connection); + LocationResponse(HttpMethod method) throws IOException { + super(method); if (isOk()) { parse(new LocationResponseHandler()); } Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -1,10 +1,26 @@ package com.amazon.s3; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.methods.DeleteMethod; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.methods.PutMethod; + /** * HTTP Methods. * * @author Elias Ross */ public enum Method { - GET, PUT, DELETE, HEAD + GET, PUT, DELETE, HEAD; + + public HttpMethod createHttpMethod() { + switch (this) { + case PUT: return new PutMethod(); + case GET: return new GetMethod(); + case DELETE: return new DeleteMethod(); + case HEAD: return new HeadMethod(); + } + throw new Error(); + } } \ No newline at end of file Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -9,12 +9,13 @@ package com.amazon.s3; -import java.net.MalformedURLException; -import java.net.URL; +import java.io.IOException; import java.security.Key; import java.util.HashMap; import java.util.Map; +import org.apache.commons.httpclient.URI; + /** * Generates URL Query Strings that can be used to perform operations. * These parameters include an expiration date, so that @@ -56,7 +57,7 @@ boolean isSecure, String server, int port) { this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, - port, CallingFormat.getSubdomainCallingFormat()); + port, CallingFormat.SUBDOMAIN); } public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey, @@ -96,32 +97,32 @@ expires = null; } - public URL create(Bucket bucket, Headers headers) + public URI create(Bucket bucket, Headers headers) { // validate bucket name if (!bucket.validateName(callingFormat)) throw new IllegalArgumentException("Invalid Bucket Name: "+bucket); - return generateURL(Method.PUT, bucket, "", headers); + return generateURI(Method.PUT, bucket, "", headers); } - public URL list(Bucket bucket, String prefix, String marker, + public URI list(Bucket bucket, String prefix, String marker, Integer maxKeys, Headers headers){ return list(bucket, prefix, marker, maxKeys, null, headers); } - public URL list(Bucket bucket, String prefix, String marker, + public URI list(Bucket bucket, String prefix, String marker, Integer maxKeys, String delimiter, Headers headers) { - return generateURL(Method.GET, bucket, headers); + return generateURI(Method.GET, bucket, headers); } - public URL delete(Bucket bucket, Headers headers) + public URI delete(Bucket bucket, Headers headers) { - return generateURL(Method.DELETE, bucket, headers); + return generateURI(Method.DELETE, bucket, headers); } - public URL put(Bucket bucket, String key, S3Object object, Headers headers) { + public URI put(Bucket bucket, String key, S3Object object, Headers headers) { Headers metadata = null; if (object != null) { metadata = object.getMetadata(); @@ -130,63 +131,61 @@ headers = new Headers(); } - return generateURL(Method.PUT, bucket, UrlEncoder.encode(key), headers.mergeMetadata(metadata)); + return generateURI(Method.PUT, bucket, key, headers.mergeMetadata(metadata)); } - public URL get(Bucket bucket, String key, Headers headers) + public URI get(Bucket bucket, String key, Headers headers) { - return generateURL(Method.GET, bucket, UrlEncoder.encode(key), headers); + return generateURI(Method.GET, bucket, key, headers); } - public URL delete(Bucket bucket, String key, Headers headers) + public URI delete(Bucket bucket, String key, Headers headers) { - return generateURL(Method.DELETE, bucket, UrlEncoder.encode(key), headers); + return generateURI(Method.DELETE, bucket, key, headers); } + + private Map<String, String> map(String name) { + HashMap<String, String> map = new HashMap<String, String>(); + map.put(name, ""); + return map; + } - public URL getBucketLogging(Bucket bucket, Headers headers) { - Map<String, String> pathArgs = new HashMap<String, String>(); - pathArgs.put("logging", null); - return generateURL(Method.GET, bucket, "", pathArgs, headers); + public URI getBucketLogging(Bucket bucket, Headers headers) { + return generateURI(Method.GET, bucket, "", map("logging"), headers); } - public URL putBucketLogging(Bucket bucket, String loggingXMLDoc, Headers headers) { - Map<String, String> pathArgs = new HashMap<String, String>(); - pathArgs.put("logging", null); - return generateURL(Method.PUT, bucket, "", pathArgs, headers); + public URI putBucketLogging(Bucket bucket, Headers headers) { + return generateURI(Method.PUT, bucket, "", map("logging"), headers); } - public URL getACL(Bucket bucket, Headers headers) { + public URI getACL(Bucket bucket, Headers headers) { return getACL(bucket, "", headers); } - public URL getACL(Bucket bucket, String key, Headers headers) + public URI getACL(Bucket bucket, String key, Headers headers) { - Map<String, String> pathArgs = new HashMap<String, String>(); - pathArgs.put("acl", null); - return generateURL(Method.GET, bucket, UrlEncoder.encode(key), pathArgs, headers); + return generateURI(Method.GET, bucket, key, map("acl"), headers); } - public URL putACL(Bucket bucket, String aclXMLDoc, Headers headers) { - return putACL(bucket, "", aclXMLDoc, headers); + public URI putACL(Bucket bucket, Headers headers) { + return putACL(bucket, "", headers); } - public URL putACL(Bucket bucket, String key, String aclXMLDoc, Headers headers) + public URI putACL(Bucket bucket, String key, Headers headers) { - Map<String, String> pathArgs = new HashMap<String, String>(); - pathArgs.put("acl", null); - return generateURL(Method.PUT, bucket, UrlEncoder.encode(key), pathArgs, headers); + return generateURI(Method.PUT, bucket, key, map("acl"), headers); } - public URL listAllBuckets(Headers headers) + public URI listAllBuckets(Headers headers) { - return generateURL(Method.GET, headers); + return generateURI(Method.GET, headers); } - public URL listAllBuckets() { + public URI listAllBuckets() { return listAllBuckets(null); } - public String makeBareURL(Bucket bucket, String key) { + public String makeBareURI(Bucket bucket, String key) { StringBuilder buffer = new StringBuilder(); if (this.isSecure) { buffer.append("https://"); @@ -200,19 +199,19 @@ } @SuppressWarnings("unchecked") - private URL generateURL(Method method, Bucket bucket, String key, Headers headers) { - return generateURL(method, bucket, key, new HashMap(), headers); + private URI generateURI(Method method, Bucket bucket, String key, Headers headers) { + return generateURI(method, bucket, key, new HashMap(), headers); } - private URL generateURL(Method method, Bucket bucket, Headers headers) { - return generateURL(method, bucket, "", headers); + private URI generateURI(Method method, Bucket bucket, Headers headers) { + return generateURI(method, bucket, "", headers); } - private URL generateURL(Method method, Headers headers) { - return generateURL(method, null, headers); + private URI generateURI(Method method, Headers headers) { + return generateURI(method, null, headers); } - private URL generateURL(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) + private URI generateURI(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) { long expires = 0L; if (this.expiresIn != null) { @@ -226,17 +225,18 @@ // convert to seconds expires /= 1000; - String canonicalString = CanonicalString.make(method, bucket, key, pathArgs, headers, ""+expires); - String encodedCanonical = CanonicalString.encode(this.awsSecretAccessKey, canonicalString, true); + String enckey = UrlEncoder.encode(key); + String canonicalString = CanonicalString.make(method, bucket, enckey, pathArgs, headers, ""+expires); + String encodedCanonical = CanonicalString.encode(this.awsSecretAccessKey, canonicalString, false); pathArgs.put("Signature", encodedCanonical); pathArgs.put("Expires", Long.toString(expires)); pathArgs.put("AWSAccessKeyId", this.awsAccessKeyId); try { - return this.callingFormat.getURL(this.isSecure, server, port, bucket, key, pathArgs); - } catch (MalformedURLException e) { - throw new IllegalStateException("Unable to generate URL " + e); + return this.callingFormat.getURI(this.isSecure, server, port, bucket, key, pathArgs); + } catch (IOException e) { + throw new IllegalStateException("Unable to generate URI " + e); } } Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -10,10 +10,10 @@ package com.amazon.s3; import java.io.IOException; -import java.net.HttpURLConnection; -import java.util.HashMap; -import java.util.Map; +import java.io.InputStream; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpMethod; import org.xml.sax.ContentHandler; import org.xml.sax.ErrorHandler; import org.xml.sax.InputSource; @@ -28,35 +28,35 @@ */ public class Response { - private HttpURLConnection connection; + private HttpMethod method; - Response(HttpURLConnection connection) throws IOException { - this.connection = connection; + Response(HttpMethod method) { + this.method = method; } /** * Returns the HTTP response code. */ public int getResponseCode() throws IOException { - return connection.getResponseCode(); + return method.getStatusCode(); } /** * Returns the HTTP response message. */ public String getResponseMessage() throws IOException { - return connection.getResponseMessage(); + return method.getStatusText(); } /** * Returns content length of the response. */ public int getContentLength() { - return connection.getContentLength(); + return -1; // TODO } /** - * Throws an IllegalStateException if not 200 OK. + * Throws an IllegalStateException if not 200-level OK. */ public void assertOk() { if (isOk()) @@ -68,7 +68,10 @@ * Returns an HTTP header field from the response. */ public String getHeaderField(String field) { - return connection.getHeaderField(field); + Header header = method.getResponseHeader(field); + if (header == null) + return null; + return header.getValue(); } /** @@ -76,7 +79,7 @@ * Prefer caching the return value. */ public Headers getHeaders() { - return new Headers(connection.getHeaderFields()); + return new Headers(method.getResponseHeaders()); } /** @@ -84,7 +87,7 @@ */ public boolean isOk() { try { - int code = connection.getResponseCode(); + int code = getResponseCode(); return code == 200 || code == 204; } catch (IOException e) { return false; @@ -96,7 +99,7 @@ */ public boolean isNotFound() { try { - return connection.getResponseCode() == 404; + return getResponseCode() == 404; } catch (IOException e) { return false; } @@ -115,30 +118,6 @@ } } - static Map<String, String> paramsForListOptions(String prefix, - String marker, Integer maxKeys) { - return paramsForListOptions(prefix, marker, maxKeys, null); - } - - static Map<String, String> paramsForListOptions(String prefix, - String marker, Integer maxKeys, String delimiter) { - - Map<String, String> argParams = new HashMap<String, String>(); - // these three params must be url encoded - if (prefix != null) - argParams.put("prefix", UrlEncoder.encode(prefix)); - if (marker != null) - argParams.put("marker", UrlEncoder.encode(marker)); - if (delimiter != null) - argParams.put("delimiter", UrlEncoder.encode(delimiter)); - - if (maxKeys != null) - argParams.put("max-keys", Integer.toString(maxKeys.intValue())); - - return argParams; - - } - /** * Parses the response. */ @@ -162,7 +141,9 @@ }; xr.setErrorHandler(eh); - xr.parse(new InputSource(connection.getInputStream())); + InputStream is = method.getResponseBodyAsStream(); + xr.parse(new InputSource(is)); + is.close(); } catch (SAXException e) { throw new RuntimeException("Unexpected error parsing ListBucket xml", e); } Modified: amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -40,6 +40,13 @@ public S3Object(byte[] data) { this(data, null); } + + /** + * Constructs a new S3Object; data is converted to bytes. + */ + public S3Object(String data) { + this(data.getBytes()); + } /** * Returns data. @@ -49,6 +56,13 @@ } /** + * Returns the length of data. + */ + public int getLength() { + return data.length; + } + + /** * Returns metadata, unmodifiable. */ public Headers getMetadata() { @@ -65,5 +79,5 @@ " data.length=" + this.data.length + " metadata=" + this.metadata + "]"; } - + } Modified: amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java =================================================================== --- amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java 2008-02-07 20:21:24 UTC (rev 5330) +++ amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java 2008-02-07 22:57:14 UTC (rev 5331) @@ -9,14 +9,17 @@ // affiliates. import java.io.IOException; +import java.io.InputStream; import java.net.HttpURLConnection; import java.net.MalformedURLException; -import java.net.URL; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.methods.EntityEnclosingMethod; +import org.apache.commons.httpclient.methods.StringRequestEntity; import org.junit.Before; import org.junit.Test; @@ -47,9 +50,9 @@ // subdomain: http://bucket.s3.amazonaws.com/key // testing pure vanity domains (http://<vanity domain>/key) is not covered here // but is possible with some assitional setup - test(CallingFormat.getSubdomainCallingFormat(), Connection.LOCATION_DEFAULT, true, Connection.DEFAULT_HOST); - test(CallingFormat.getPathCallingFormat(), Connection.LOCATION_DEFAULT, true, Connection.DEFAULT_HOST); - test(CallingFormat.getSubdomainCallingFormat(), Connection.LOCATION_EU, true, Connection.DEFAULT_HOST); + test(CallingFormat.SUBDOMAIN, Connection.LOCATION_DEFAULT, false, Connection.DEFAULT_HOST); + test(CallingFormat.PATH, Connection.LOCATION_DEFAULT, true, Connection.DEFAULT_HOST); + test(CallingFormat.SUBDOMAIN, Connection.LOCATION_EU, true, Connection.DEFAULT_HOST); } private void test(CallingFormat format, String location, boolean secure, String server) throws Exception @@ -63,17 +66,17 @@ new QueryGenerator(awsAccessKeyId, awsSecretAccessKey, secure, server, format); Response response = conn.create(bucket, location, null); - assertEquals( - "couldn't create bucket", - HttpURLConnection.HTTP_OK, - response.getResponseCode()); + response.assertOk(); - ListResponse listBucketResponse = conn.list(bucket, null, null, null, null); - assertEquals( - "couldn't get list", - HttpURLConnection.HTTP_OK, - listBucketResponse.getResponseCode()); - assertEquals("list wasn't empty", 0, listBucketResponse.getEntries().size()); + ListResponse listBucketResponse = conn.list(bucket); + listBucketResponse.assertOk(); + for (Entry entry : listBucketResponse.getEntries()) { + Response delete = conn.delete(bucket, entry.getKey()); + System.out.println("DEL " + delete); + delete.assertOk(); + } + listBucketResponse = conn.list(bucket); + assertEquals("list wasn't empty " + listBucketResponse, 0, listBucketResponse.getEntries().size()); System.out.println(listBucketResponse); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); @@ -84,79 +87,54 @@ final String innerKey = "test/inner.txt"; final String lastKey = "z-last-key.txt"; - response = conn.put(bucket, key, new S3Object(text.getBytes())); - assertEquals( - "couldn't put simple object", - HttpURLConnection.HTTP_OK, - response.getResponseCode()); + response = conn.put(bucket, key, new S3Object(text)); + response.assertOk(); - response = conn.put(bucket, innerKey, new S3Object(text.getBytes()), null); - assertEquals( - "couldn't put simple object", - HttpURLConnection.HTTP_OK, - response.getResponseCode()); + response = conn.put(bucket, innerKey, new S3Object(text)); + response.assertOk(); - response = conn.put(bucket, lastKey, new S3Object(text.getBytes()), null); - assertEquals( - "couldn't put simple object", - HttpURLConnection.HTTP_OK, - response.getResponseCode()); + response = conn.put(bucket, lastKey, new S3Object(text)); + response.assertOk(); // plain list - listBucketResponse = conn.list(bucket, null, null, null, null); - assertEquals( - "couldn't get list", - listBucketResponse.getResponseCode(), - HttpURLConnection.HTTP_OK); + listBucketResponse = conn.list(bucket); + listBucketResponse.assertOk(); assertEquals("Unexpected list size", 3, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); + + System.out.println("LIST " + listBucketResponse.getEntries()); // root "directory" listBucketResponse = conn.list(bucket, null, null, null, "/", null); - assertEquals( - "couldn't get list", - HttpURLConnection.HTTP_OK, - listBucketResponse.getResponseCode()); - assertEquals("Unexpected list size", 2, listBucketResponse.getEntries().size()); + listBucketResponse.assertOk(); + assertEquals("Unexpected list size " + listBucketResponse, 2, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 1, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, "/", false, null); // root "directory" with a max-keys of "1" listBucketResponse = conn.list(bucket, null, null, 1, "/", null); - assertEquals( - "couldn't get list", - HttpURLConnection.HTTP_OK, - listBucketResponse.getResponseCode()); + listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 1, "/", true, "example.txt"); // root "directory" with a max-keys of "2" listBucketResponse = conn.list(bucket, null, null, 2, "/", null); - assertEquals( - "couldn't get list", - HttpURLConnection.HTTP_OK, - listBucketResponse.getResponseCode()); + listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 1, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 2, "/", true, "test/"); String marker = listBucketResponse.getNextMarker(); listBucketResponse = conn.list(bucket, null, marker, new Integer( 2 ), "/", null); - assertEquals( - "couldn't get list", - HttpURLConnection.HTTP_OK, - listBucketResponse.getResponseCode()); + listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", marker, 2, "/", false, null); // test "directory" listBucketResponse = conn.list(bucket, "test/", null, null, "/", null); - assertEquals( - "couldn't get list", - HttpURLConnection.HTTP_OK, - listBucketResponse.getResponseCode()); + listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "test/", "", UnspecifiedMaxKeys, "/", false, null); @@ -179,24 +157,15 @@ // end delimiter tests response = conn.put(bucket, key, new S3Object(text.getBytes(), null), null); - assertEquals( - "couldn't reput simple object", - HttpURLConnection.HTTP_OK, - response.getResponseCode()); + response.assertOk(); - Map<String, List<String>> metadata = new HashMap<String, List<String>>(); - metadata.put("title", Arrays.asList(new String[] { "title" })); - response = conn.put(bucket, key, new S3Object(text.getBytes(), new Headers(metadata)), null); - assertEquals( - "couldn't put complex object", - HttpURLConnection.HTTP_OK, - response.getResponseCode()); + Headers metadata = new Headers(); + metadata.put("title", "title"); + response = conn.put(bucket, key, new S3Object(text.getBytes(), metadata), null); + response.assertOk(); GetResponse getResponse = conn.get(bucket, key, null); - assertEquals( - "couldn't get object", - getResponse.getResponseCode(), - HttpURLConnection.HTTP_OK); + getResponse.assertOk(); assertEquals("didn't get the right data back", text.getBytes(), getResponse.getObject().getData()); assertEquals("didn't get the right metadata back", 1, getResponse.getObject().getMetadata().size()); assertEquals( @@ -207,8 +176,14 @@ "didn't get the right content-length", ""+text.length(), getResponse.getHeaderField("Content-Length")); + + GetStreamResponse streamResponse = conn.getStream(bucket, key); + InputStream is = streamResponse.getInputStream(); + byte b[] = new byte[text.length()]; + int len = is.read(b); + assertEquals("didn't get the right data back " + len, text.getBytes(), b); + streamResponse.release(); - String titleWithSpaces = " \t title with leading and trailing spaces "; Headers h = new Headers(); h.put("title", titleWithSpaces); @@ -345,73 +320,73 @@ buckets.size() - 1, listAllMyBucketsResponse.getEntries().size()); - checkURL( + checkURI( generator.create(bucket, null), - "PUT", + Method.PUT, HttpURLConnection.HTTP_OK, "couldn't create bucket"); - checkURL( + checkURI( generator.put(bucket, key, new S3Object("test data".getBytes(), null), null), - "PUT", + Method.PUT, HttpURLConnection.HTTP_OK, "put object", "test data"); - checkURL( + checkURI( generator.get(bucket, key, null), - "GET", + Method.GET, HttpURLConnection.HTTP_OK, "get object"); - checkURL( + checkURI( generator.list(bucket, null, null, null, null), - "GET", + Method.GET, HttpURLConnection.HTTP_OK, "list bucket"); - checkURL( + checkURI( generator.listAllBuckets(), - "GET", + Method.GET, HttpURLConnection.HTTP_OK, "list all my buckets"); - checkURL( + checkURI( generator.getACL(bucket, key, null), - "GET", + Method.GET, HttpURLConnection.HTTP_OK, "get acl"); - checkURL( - generator.putACL(bucket, key, new String(acl), null), - "PUT", + checkURI( + generator.putACL(bucket, key, null), + Method.PUT, HttpURLConnection.HTTP_OK, "put acl", new String(acl)); - checkURL( + checkURI( generator.getACL(bucket, null), - "GET", + Method.GET, HttpURLConnection.HTTP_OK, "get bucket acl"); - checkURL( - generator.putACL(bucket, new String(bucketACL), null), - "PUT", + checkURI( + generator.putACL(bucket, null), + Method.PUT, HttpURLConnection.HTTP_OK, "put bucket acl", new String(bucketACL)); - checkURL( + checkURI( generator.getBucketLogging(bucket, null), - "GET", + Method.GET, HttpURLConnection.HTTP_OK, "get bucket logging"); - checkURL( - generator.putBucketLogging(bucket, new String(bucketLogging), null), - "PUT", + checkURI( + generator.putBucketLogging(bucket, null), + Method.PUT, HttpURLConnection.HTTP_OK, "put bucket logging", new String(bucketLogging)); - checkURL( + checkURI( generator.delete(bucket, key, null), - "DELETE", + Method.DELETE, HttpURLConnection.HTTP_NO_CONTENT, "delete object"); - checkURL( + checkURI( generator.delete(bucket, null), - "DELETE", + Method.DELETE, HttpURLConnection.HTTP_NO_CONTENT, "delete bucket"); } @@ -461,29 +436,27 @@ } } - private static void checkURL(URL url, String method, int code, String message) + private static void checkURI(URI uri, Method method, int code, String message) throws MalformedURLException, IOException { - checkURL(url, method, code, message, null); + checkURI(uri, method, code, message, null); } - private static void checkURL(URL url, String method, int code, String message, String data) + private static void checkURI(URI uri, Method method, int code, String message, String data) throws MalformedURLException, IOException { if (data == null) data = ""; - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod(method); - if ("PUT".equals(method)) { - connection.setRequestProperty("Content-Length", ""+data.getBytes().length); - connection.setDoOutput(true); - connection.getOutputStream().write(data.getBytes()); - } else { - // HttpURLConnection auto populates Content-Type, which we don't want here - connection.setRequestProperty("Content-Type", ""); + HttpClient client = new HttpClient(); + HttpMethod httpMethod = method.createHttpMethod(); + if (method == Method.PUT) { + ((EntityEnclosingMethod) httpMethod).setRequestEntity(new StringRequestEntity(data)); } + httpMethod.setURI(uri); + int response = client.executeMethod(httpMethod); + httpMethod.releaseConnection(); - assertEquals(message, code, connection.getResponseCode()); + assertEquals(message, code, response); } private void readline() throws IOException { @@ -546,7 +519,7 @@ readline(); System.out.println("\nNow try just the url without the query string arguments. It should fail.\n"); - System.out.println(generator.makeBareURL(bucket, keyName)); + System.out.println(generator.makeBareURI(bucket, keyName)); System.out.print("\npress enter> "); readline(); @@ -566,14 +539,14 @@ System.out.println("----- anonymous read test -----"); System.out.println("\nYou should be able to try this in your browser\n"); - System.out.println(generator.makeBareURL(bucket, keyName + "-public")); + System.out.println(generator.makeBareURI(bucket, keyName + "-public")); System.out.print("\npress enter> "); readline(); System.out.println("----- path style url example -----"); System.out.println("\nNon-location-constrained buckets can also be specified as part of the url path. (This was the original url style supported by S3.)"); System.out.println("\nTry this url out in your browser (it will only be valid for 60 seconds)\n"); - generator.setCallingFormat(CallingFormat.getPathCallingFormat()); + generator.setCallingFormat(CallingFormat.PATH); // could also have been done like this: // generator = new QueryStringAuthGenerator(awsAccessKeyId, awsSecretAccessKey, true, Utils.DEFAULT_HOST, CallingFormat.getPathCallingFormat()); generator.setExpiresIn(60 * 1000); Added: amazon-s3/trunk/src/test/resources/log4j.xml =================================================================== --- amazon-s3/trunk/src/test/resources/log4j.xml (rev 0) +++ amazon-s3/trunk/src/test/resources/log4j.xml 2008-02-07 22:57:14 UTC (rev 5331) @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<!-- ===================================================================== --> +<!-- --> +<!-- Log4j Configuration --> +<!-- --> +<!-- ===================================================================== --> + +<!-- $Id: log4j.xml,v 1.2 2008/01/24 17:52:51 rosse Exp $ --> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" debug="false"> + + <!-- ============================== --> + <!-- Append messages to the console --> + <!-- ============================== --> + + <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + + <layout class="org.apache.log4j.PatternLayout"> + <!-- The default pattern: Date Priority [Category] Message\n --> + <param name="ConversionPattern" value="%r %-5p [%c{1}] (%t) %m%n"/> + </layout> + </appender> + + <logger name="org.apache"> + <level value="INFO"/> + </logger> + <logger name="httpclient.wire"> + <level value="INFO"/> + </logger> + <logger name="org.apache.commons.httpclient"> + <level value="INFO"/> + </logger> + + <!-- ======================= --> + <!-- Setup the Root category --> + <!-- ======================= --> + + <root> + <priority value="INFO"/> + <appender-ref ref="CONSOLE"/> + </root> + +</log4j:configuration>