[jbosscache-commits] JBoss Cache SVN: r5305 - in amazon-s3/trunk: src and 11 other directories.

Author: genman Date: 2008-02-05 20:05:04 -0500 (Tue, 05 Feb 2008) New Revision: 5305 Added: amazon-s3/trunk/.classpath amazon-s3/trunk/.project amazon-s3/trunk/pom.xml amazon-s3/trunk/src/ amazon-s3/trunk/src/main/ amazon-s3/trunk/src/main/java/ amazon-s3/trunk/src/main/java/com/ amazon-s3/trunk/src/main/java/com/amazon/ amazon-s3/trunk/src/main/java/com/amazon/s3/ amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java amazon-s3/trunk/src/main/java/com/amazon/s3/Entry.java amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java amazon-s3/trunk/src/main/java/com/amazon/s3/Owner.java amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java amazon-s3/trunk/src/main/java/com/amazon/s3/UrlEncoder.java amazon-s3/trunk/src/main/java/com/amazon/thirdparty/ amazon-s3/trunk/src/main/java/com/amazon/thirdparty/Base64.java amazon-s3/trunk/src/test/ amazon-s3/trunk/src/test/java/ amazon-s3/trunk/src/test/java/com/ amazon-s3/trunk/src/test/java/com/amazon/ amazon-s3/trunk/src/test/java/com/amazon/s3/ amazon-s3/trunk/src/test/java/com/amazon/s3/S3Driver.java amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java Log: Cleaned-up Amazon S3 REST client/tests For experimental class loader Added: amazon-s3/trunk/.classpath =================================================================== --- amazon-s3/trunk/.classpath (rev 0) +++ amazon-s3/trunk/.classpath 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src/main/java"/> + <classpathentry kind="src" path="src/test/java"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> + <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/> + <classpathentry kind="output" path="bin"/> +</classpath> Added: amazon-s3/trunk/.project =================================================================== --- amazon-s3/trunk/.project (rev 0) +++ amazon-s3/trunk/.project 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8"?> +<projectDescription> + <name>s3-rest</name> + <comment></comment> + <projects> + </projects> + <buildSpec> + <buildCommand> + <name>org.maven.ide.eclipse.maven2Builder</name> + <arguments> + </arguments> + </buildCommand> + <buildCommand> + <name>org.eclipse.jdt.core.javabuilder</name> + <arguments> + </arguments> + </buildCommand> + </buildSpec> + <natures> + <nature>org.eclipse.jdt.core.javanature</nature> + <nature>org.maven.ide.eclipse.maven2Nature</nature> + </natures> +</projectDescription> Added: amazon-s3/trunk/pom.xml =================================================================== --- amazon-s3/trunk/pom.xml (rev 0) +++ amazon-s3/trunk/pom.xml 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?><project> + <modelVersion>4.0.0</modelVersion> + <groupId>com.amazon</groupId> + <artifactId>amazon-s3-rest</artifactId> + <version>1.0.0.0-SNAPSHOT</version> + <description> + Cleaned-up version of the Amazon S3 REST access layer + </description> +</project> \ No newline at end of file Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Bucket.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,127 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.util.Date; +import java.util.regex.Pattern; + +/** + * A class representing a single bucket. + */ +public class Bucket { + + /** + * The name of the bucket. + */ + private String name; + + /** + * The bucket's creation date. + */ + private Date creationDate; + + Bucket() { + } + + /** + * Constructs a new Bucket. + * + * @param name non-null name + */ + public Bucket(String name) { + if (name == null) + throw new NullPointerException(); + this.name = name; + } + + /** + * Returns name. + */ + public String getName() { + return name; + } + + /** + * Returns true if specified; length is non-zero. + */ + public boolean specified() { + return name.length() != 0; + } + + /** + * Sets name. + */ + void setName(String name) { + this.name = name; + } + + /** + * Returns creationDate. + */ + public Date getCreationDate() { + return creationDate; + } + + /** + * Sets creationDate. + */ + void setCreationDate(Date creationDate) { + this.creationDate = creationDate; + } + + /** + * Returns the name. + */ + @Override + public String toString() { + return name; + } + + static Pattern bucketPath = Pattern.compile("^[0-9A-Za-z\\.\\-_]*$"); + static Pattern ipv4Pattern = Pattern.compile("^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$"); + static Pattern bucketNamePattern = Pattern.compile("^[a-z0-9]([a-z0-9\\-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9\\-]*[a-z0-9])?)*$"); + + /** + * Returns true if the bucket name is valid for the calling format. + */ + public static boolean validateBucketName(String bucketName, CallingFormat callingFormat) { + if (callingFormat == CallingFormat.getPathCallingFormat()) + { + final int MIN_BUCKET_LENGTH = 3; + final int MAX_BUCKET_LENGTH = 255; + + return null != bucketName && + bucketName.length() >= MIN_BUCKET_LENGTH && + bucketName.length() <= MAX_BUCKET_LENGTH && + bucketPath.matcher(bucketName).matches(); + } else { + final int MIN_BUCKET_LENGTH = 3; + final int MAX_BUCKET_LENGTH = 63; + + // If there wasn't a location-constraint, then the current actual + // restriction is just that no 'part' of the name (i.e. sequence + // of characters between any 2 '.'s has to be 63) but the recommendation + // is to keep the entire bucket name under 63. + return null != bucketName && + bucketName.length() >= MIN_BUCKET_LENGTH && + bucketName.length() <= MAX_BUCKET_LENGTH && + !ipv4Pattern.matcher(bucketName).matches() && + bucketNamePattern.matcher(bucketName).matches(); + } + } + + /** + * Returns true if the bucket name is valid for the calling format. + */ + public boolean validateName(CallingFormat callingFormat) { + return validateBucketName(name, callingFormat); + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/CallingFormat.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,149 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; + +/** + * Calling formats. + * + * @author Elias Ross + */ +public abstract class CallingFormat { + + protected static CallingFormat pathCallingFormat = new PathCallingFormat(); + protected static CallingFormat subdomainCallingFormat = new SubdomainCallingFormat(); + protected static CallingFormat vanityCallingFormat = new VanityCallingFormat(); + + public abstract boolean supportsLocatedBuckets(); + + public abstract String getEndpoint(String server, int port, Bucket bucket); + + public abstract String getPathBase(Bucket bucket, String key); + + public abstract URL getURL(boolean isSecure, String server, int port, + Bucket bucket, String key, Map<String, String> pathArgs) + throws MalformedURLException; + + /** + * Converts the Path Arguments from a map to String which can be used in url construction + * @param pathArgs a map of arguments + * @return a string representation of pathArgs + */ + public static String convertPathArgsHashToString(Map<String, String> pathArgs) { + StringBuilder pathArgsString = new StringBuilder(); + String argumentValue; + boolean firstRun = true; + if (pathArgs != null) { + for (Iterator<String> argumentIterator = pathArgs.keySet().iterator(); argumentIterator.hasNext(); ) { + String argument = argumentIterator.next(); + if (firstRun) { + firstRun = false; + pathArgsString.append("?"); + } else { + pathArgsString.append("&"); + } + + argumentValue = pathArgs.get(argument); + pathArgsString.append(argument); + if (argumentValue != null) { + pathArgsString.append("="); + pathArgsString.append(argumentValue); + } + } + } + + return pathArgsString.toString(); + } + + public static CallingFormat getPathCallingFormat() { + return pathCallingFormat; + } + + public static CallingFormat getSubdomainCallingFormat() { + return subdomainCallingFormat; + } + + public static CallingFormat getVanityCallingFormat() { + return vanityCallingFormat; + } + + static private class PathCallingFormat extends CallingFormat { + public boolean supportsLocatedBuckets() { + return false; + } + + public String getPathBase(Bucket bucket, String key) { + return isBucketSpecified(bucket) ? "/" + bucket + "/" + key : "/"; + } + + public String getEndpoint(String server, int port, Bucket bucket) { + return server + ":" + port; + } + + public URL getURL(boolean isSecure, String server, int port, Bucket bucket, String key, Map<String, String> pathArgs) + throws MalformedURLException { + String pathBase = isBucketSpecified(bucket) ? "/" + bucket + "/" + key : "/"; + String pathArguments = convertPathArgsHashToString(pathArgs); + return new URL(isSecure ? "https" : "http", server, port, pathBase + pathArguments); + } + + private boolean isBucketSpecified(Bucket bucket) { + if (bucket == null) + return false; + if (bucket.getName().length() == 0) + return false; + return true; + } + } + + static private class SubdomainCallingFormat extends CallingFormat { + public boolean supportsLocatedBuckets() { + return true; + } + + public String getServer(String server, Bucket bucket) { + return bucket + "." + server; + } + + public String getEndpoint(String server, int port, Bucket bucket) { + return getServer(server, bucket) + ":" + port; + } + + public String getPathBase(Bucket bucket, String key) { + return "/" + key; + } + + public URL getURL(boolean isSecure, String server, int port, + Bucket bucket, String key, Map<String, String> pathArgs) + throws MalformedURLException { + if (bucket == null || !bucket.specified()) { + // The bucket is null, this is listAllBuckets request + String pathArguments = convertPathArgsHashToString(pathArgs); + return new URL(isSecure ? "https" : "http", server, port, "/" + + pathArguments); + } else { + String serverToUse = getServer(server, bucket); + String pathBase = getPathBase(bucket, key); + String pathArguments = convertPathArgsHashToString(pathArgs); + return new URL(isSecure ? "https" : "http", serverToUse, port, pathBase + pathArguments); + } + } + } + + static private class VanityCallingFormat extends SubdomainCallingFormat { + public String getServer(String server, Bucket bucket) { + return bucket.getName(); + } + } +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/CanonicalString.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,199 @@ +package com.amazon.s3; + +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import com.amazon.thirdparty.Base64; + +/** + * Creates canonical strings for authorization purposes by hashing + * the request against an authorization key. + * + * @author Elias Ross + */ +public class CanonicalString { + + private static final String AMAZON_HEADER_PREFIX = "x-amz-"; + private static final String ALTERNATIVE_DATE_HEADER = "x-amz-date"; + + /** + * HMAC/SHA1 Algorithm per RFC 2104. + */ + private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; + + private CanonicalString() { + } + + /** + * Returns a canonical string used in authentication. + */ + public static String make(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) { + return make(method, bucket, key, pathArgs, headers, null); + } + + /** + * Returns a canonical string used in authentication. + * + * @param expires When non-null, it will be used instead of the Date header. + * @param key URL-encoded string + */ + public static String make(Method method, Bucket bucket, String key, Map<String, String> pathArgs, + Headers headers, String expires) + { + StringBuilder buf = new StringBuilder(); + buf.append(method + "\n"); + + // Add all interesting headers to a list, then sort them. "Interesting" + // is defined as Content-MD5, Content-Type, Date, and x-amz- + SortedMap<String, String> interestingHeaders = new TreeMap<String, String>(); + if (headers != null) { + for (Map.Entry<String, List<String>> me : headers.getHeaders().entrySet()) { + String hashKey = me.getKey(); + if (hashKey == null) + continue; + String lk = hashKey.toLowerCase(Locale.US); + + // Ignore any headers that are not particularly interesting. + if (lk.equals("content-type") || lk.equals("content-md5") || lk.equals("date") || + lk.startsWith(AMAZON_HEADER_PREFIX)) + { + interestingHeaders.put(lk, concatenateList(me.getValue())); + } + } + } + + if (interestingHeaders.containsKey(ALTERNATIVE_DATE_HEADER)) { + interestingHeaders.put("date", ""); + } + + // if the expires is non-null, use that for the date field. this + // trumps the x-amz-date behavior. + if (expires != null) { + interestingHeaders.put("date", expires); + } + + // these headers require that we still put a new line in after them, + // even if they don't exist. + if (! interestingHeaders.containsKey("content-type")) { + interestingHeaders.put("content-type", ""); + } + if (! interestingHeaders.containsKey("content-md5")) { + interestingHeaders.put("content-md5", ""); + } + + // Finally, add all the interesting headers (i.e.: all that startwith x-amz- ;-)) + for (Iterator<String> i = interestingHeaders.keySet().iterator(); i.hasNext(); ) { + String headerKey = i.next(); + if (headerKey.startsWith(AMAZON_HEADER_PREFIX)) { + buf.append(headerKey).append(':').append(interestingHeaders.get(headerKey)); + } else { + buf.append(interestingHeaders.get(headerKey)); + } + buf.append("\n"); + } + + // build the path using the bucket and key + if (bucket != null && bucket.specified()) { + buf.append("/" + bucket.getName() ); + } + + // append the key (it might be an empty string) + // append a slash regardless + buf.append("/"); + if(key != null) { + buf.append(key); + } + + // if there is an acl, logging or torrent parameter + // add them to the string + if (pathArgs != null ) { + if (pathArgs.containsKey("acl")) { + buf.append("?acl"); + } else if (pathArgs.containsKey("torrent")) { + buf.append("?torrent"); + } else if (pathArgs.containsKey("logging")) { + buf.append("?logging"); + } else if (pathArgs.containsKey("location")) { + buf.append("?location"); + } + } + + return buf.toString(); + } + + /** + * Concatenates a bunch of header values, separating them with a comma. + * @param values List of header values. + * @return String of all headers, with commas. + */ + private static String concatenateList(List<String> values) { + StringBuilder buf = new StringBuilder(); + for (int i = 0, size = values.size(); i < size; ++ i) { + buf.append(values.get(i).replaceAll("\n", "").trim()); + if (i != (size - 1)) { + buf.append(","); + } + } + return buf.toString(); + } + + /** + * Returns an encrypted key for the access key. + */ + static Key key(String awsSecretAccessKey) { + // The following HMAC/SHA1 code for the signature is taken from the + // AWS Platform's implementation of RFC2104 (amazon.webservices.common.Signature) + // + // Acquire an HMAC/SHA1 from the raw key bytes. + SecretKeySpec signingKey = + new SecretKeySpec(awsSecretAccessKey.getBytes(), HMAC_SHA1_ALGORITHM); + return signingKey; + } + + /** + * Calculate the HMAC/SHA1 on a string. + * @param data Data to sign + * @param passcode Passcode to sign it with + * @return Signature + * @throws NoSuchAlgorithmException If the algorithm does not exist. Unlikely + * @throws InvalidKeyException If the key is invalid. + */ + static String encode(Key signingKey, String canonicalString, + boolean urlencode) + { + // Acquire the MAC instance and initialize with the signing key. + Mac mac = null; + try { + mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); + } catch (NoSuchAlgorithmException e) { + // should not happen + throw new RuntimeException("Could not find sha1 algorithm", e); + } + try { + mac.init(signingKey); + } catch (InvalidKeyException e) { + // also should not happen + throw new RuntimeException("Could not initialize the MAC algorithm", e); + } + + // Compute the HMAC on the digest, and set it. + String b64 = Base64.encodeBytes(mac.doFinal(canonicalString.getBytes())); + + if (urlencode) { + return UrlEncoder.encode(b64); + } else { + return b64; + } + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Connection.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,579 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.Key; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + +/** + * A stateless connection to the Amazon S3 system which uses the REST API. + * <p/> + * It is initially configured with authentication and connection parameters and + * exposes methods to access and manipulate S3 data. + */ +public class Connection { + + public static final String LOCATION_DEFAULT = null; + public static final String LOCATION_EU = "EU"; + + public static final String DEFAULT_HOST = "s3.amazonaws.com"; + public static final int INSECURE_PORT = 80; + public static final int SECURE_PORT = 443; + + private String awsAccessKeyId; + private Key awsSecretAccessKey; + private boolean isSecure; + private String server; + private int port; + private CallingFormat callingFormat; + + public Connection(String awsAccessKeyId, String awsSecretAccessKey) { + this(awsAccessKeyId, awsSecretAccessKey, true); + } + + public Connection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure) { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, DEFAULT_HOST); + } + + public Connection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, + String server) + { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, + isSecure ? SECURE_PORT : INSECURE_PORT); + } + + public Connection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, + String server, int port) { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, port, CallingFormat.getSubdomainCallingFormat()); + + } + + public Connection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, + String server, CallingFormat format) { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, + isSecure ? SECURE_PORT : INSECURE_PORT, + CallingFormat.getSubdomainCallingFormat()); + } + + /** + * Create a new interface to interact with S3 with the given credential and connection + * parameters + * + * @param awsAccessKeyId Your user key into AWS + * @param awsSecretAccessKey The secret string used to generate signatures for authentication. + * @param isSecure use SSL encryption + * @param server Which host to connect to. Usually, this will be s3.amazonaws.com + * @param port Which port to use. + * @param callingFormat Type of request Regular/Vanity or Pure Vanity domain + */ + public Connection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, + String server, int port, CallingFormat format) + { + this.awsAccessKeyId = awsAccessKeyId; + this.awsSecretAccessKey = CanonicalString.key(awsSecretAccessKey); + this.isSecure = isSecure; + this.server = server; + this.port = port; + this.callingFormat = format; + } + + /** + * Creates a new bucket. + * @param bucket The name of the bucket to create. + * @param location Desired location ("EU") (or null for default). + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + * @param metadata A Map of String to List of Strings representing the s3 + * metadata for this bucket (can be null). + * @throws IllegalArgumentException on invalid location + */ + public Response create(Bucket bucket, String location, Headers headers) + throws IOException + { + String body; + if (location == null) { + body = null; + } else if (LOCATION_EU.equals(location)) { + if (!callingFormat.supportsLocatedBuckets()) + throw new IllegalArgumentException("Creating location-constrained bucket with unsupported calling-format"); + body = "<CreateBucketConstraint><LocationConstraint>" + location + "</LocationConstraint></CreateBucketConstraint>"; + } else + throw new IllegalArgumentException("Invalid Location: "+location); + + // validate bucket name + if (!bucket.validateName(callingFormat)) + throw new IllegalArgumentException("Invalid Bucket Name: "+bucket); + + HttpURLConnection request = makeRequest(Method.PUT, bucket, headers); + if (body != null) + { + request.setDoOutput(true); + request.getOutputStream().write(body.getBytes("UTF-8")); + } + return new Response(request); + } + + /** + * Creates a new bucket. + * @throws IOException + */ + public Response create(Bucket bucket, String location) throws IOException { + return create(bucket, location, null); + } + + /** + * Check if the specified bucket exists (via a HEAD request) + * @param bucket The name of the bucket to check + * @return true if HEAD access returned success + */ + public boolean exists(Bucket bucket) throws IOException + { + HttpURLConnection response = makeRequest(Method.HEAD, bucket); + int httpCode = response.getResponseCode(); + return httpCode >= 200 && httpCode < 300; + } + + /** + * Lists the contents of a bucket. + * @param bucket The name of the bucket + * @param prefix All returned keys will start with this string (can be null). + * @param marker All returned keys will be lexographically greater than + * this string (can be null). + * @param maxKeys The maximum number of keys to return (can be null). + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public ListResponse list(Bucket bucket, String prefix, String marker, + Integer maxKeys, Headers headers) + throws IOException + { + return list(bucket, prefix, marker, maxKeys, null, headers); + } + + /** + * Lists the contents of a bucket. + */ + public ListResponse list(Bucket bucket, String prefix, String marker, + Integer maxKeys) throws IOException + { + return list(bucket, prefix, marker, maxKeys, null); + } + + /** + * Lists the contents of a bucket. + */ + public ListResponse list(Bucket bucket, Integer maxKeys) throws IOException + { + return list(bucket, null, null, maxKeys); + } + + /** + * Lists the contents of a bucket. + */ + public ListResponse list(Bucket bucket) throws IOException + { + return list(bucket, null, null, null, null); + } + + /** + * Lists the contents of a bucket. + * @param bucket The name of the bucket to list. + * @param prefix All returned keys will start with this string (can be null). + * @param marker All returned keys will be lexographically greater than + * this string (can be null). + * @param maxKeys The maximum number of keys to return (can be null). + * @param delimiter Keys that contain a string between the prefix and the first + * occurrence of the delimiter will be rolled up into a single element. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public ListResponse list(Bucket bucket, String prefix, String marker, + Integer maxKeys, String delimiter, Headers headers) + throws IOException + { + + Map<String, String> pathArgs = Response.paramsForListOptions(prefix, marker, maxKeys, delimiter); + return new ListResponse(makeRequest(Method.GET, bucket, pathArgs, headers)); + } + + /** + * Deletes a bucket. + * @param bucket The name of the bucket to delete. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public Response delete(Bucket bucket, Headers headers) + throws IOException + { + return new Response(makeRequest(Method.DELETE, bucket, "", null, headers)); + } + + /** + * Deletes a bucket. + */ + public Response delete(Bucket bucket) + throws IOException + { + return delete(bucket, (Headers)null); + } + + /** + * Writes an object to S3. + * @param bucket The name of the bucket to which the object will be added. + * @param key The name of the key to use. + * @param object An S3Object containing the data to write. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public Response put(Bucket bucket, String key, S3Object object, Headers headers) + throws IOException + { + HttpURLConnection request = + makeRequest(Method.PUT, bucket, key, null, headers, object); + + request.setDoOutput(true); + request.getOutputStream().write(object.getData() == null ? new byte[] {} : object.getData()); + + return new Response(request); + } + + public Response put(Bucket bucket, String key, S3Object object) { + return put(bucket, key, object); + } + + /** + * Reads an object from S3. + * @param bucket The name of the bucket where the object lives. + * @param key The name of the key to use. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public GetResponse get(Bucket bucket, String key, Headers headers) + throws IOException + { + return new GetResponse(makeRequest(Method.GET, bucket, key, null, headers)); + } + + public GetResponse get(Bucket bucket, String key) + { + return get(bucket, key); + } + + /** + * Deletes an object from S3. + * @param bucket The name of the bucket where the object lives. + * @param key The name of the key to use. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public Response delete(Bucket bucket, String key, Headers headers) + throws IOException + { + return new Response(makeRequest(Method.DELETE, bucket, key, null, headers)); + } + + /** + * Deletes an object from S3. + */ + public Response delete(Bucket bucket, String key) + { + return delete(bucket, key); + } + + /** + * Get the logging xml document for a given bucket + * @param bucket The name of the bucket + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public GetResponse getBucketLogging(Bucket bucket, Headers headers) + throws IOException + { + Map<String, String> pathArgs = Collections.singletonMap("logging", null); + return new GetResponse(makeRequest(Method.GET, bucket, "", pathArgs, headers)); + } + + /** + * Write a new logging xml document for a given bucket + * @param loggingXMLDoc The xml representation of the logging configuration as a String + * @param bucket The name of the bucket + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public Response putBucketLogging(Bucket bucket, String loggingXMLDoc, Headers headers) + throws IOException + { + Map<String, String> pathArgs = Collections.singletonMap("logging", null); + S3Object object = new S3Object(loggingXMLDoc.getBytes(), null); + HttpURLConnection request = makeRequest(Method.PUT, bucket, "", pathArgs, headers, object); + + request.setDoOutput(true); + request.getOutputStream().write(object.getData() == null ? new byte[] {} : object.getData()); + + return new Response(request); + } + + /** + * Get the ACL for a given bucket + * @param bucket The name of the bucket where the object lives. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public GetResponse getACL(Bucket bucket, Headers headers) + throws IOException + { + return getACL(bucket, "", headers); + } + + /** + * Get the ACL for a given object (or bucket, if key is null). + * @param bucket The name of the bucket where the object lives. + * @param key The name of the key to use. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public GetResponse getACL(Bucket bucket, String key, Headers headers) + throws IOException + { + if (key == null) + key = ""; + + Map<String, String> pathArgs = Collections.singletonMap("acl", null); + return new GetResponse( + makeRequest(Method.GET, bucket, key, pathArgs, headers) + ); + } + + /** + * Write a new ACL for a given bucket + * @param aclXMLDoc The xml representation of the ACL as a String + * @param bucket The name of the bucket where the object lives. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public Response putACL(Bucket bucket, String aclXMLDoc, Headers headers) + throws IOException + { + return putACL(bucket, "", aclXMLDoc, headers); + } + + /** + * Write a new ACL for a given object + * @param aclXMLDoc The xml representation of the ACL as a String + * @param bucket The name of the bucket where the object lives. + * @param key The name of the key to use. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public Response putACL(Bucket bucket, String key, String aclXMLDoc, Headers headers) + throws IOException + { + S3Object object = new S3Object(aclXMLDoc.getBytes(), null); + + Map<String, String> pathArgs = new HashMap<String, String>(); + pathArgs.put("acl", null); + + HttpURLConnection request = + makeRequest(Method.PUT, bucket, key, pathArgs, headers, object); + + request.setDoOutput(true); + request.getOutputStream().write(object.getData() == null ? new byte[] {} : object.getData()); + + return new Response(request); + } + + /** + * Returns the bucket location. + */ + public LocationResponse getLocation(Bucket bucket) + throws IOException + { + Map<String, String> pathArgs = new HashMap<String, String>(); + pathArgs.put("location", null); + return new LocationResponse(makeRequest(Method.GET, bucket, "", pathArgs, null)); + } + + + /** + * Lists all the buckets created by this account. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + public ListAllBucketsResponse listAllBuckets(Headers headers) + throws IOException + { + return new ListAllBucketsResponse(makeRequest(Method.GET, null, "", null, headers)); + } + + /** + * Lists all the buckets created by this account. + */ + public ListAllBucketsResponse listAllBuckets() throws IOException + { + return listAllBuckets(null); + } + + /** + * Make a new HttpURLConnection without passing an S3Object parameter. + * Use this method for key operations that do require arguments + * @param method The method to invoke + * @param bucketName the bucket this request is for + * @param key the key this request is for + * @param pathArgs the + * @param headers + * @return + * @throws IOException + */ + private HttpURLConnection makeRequest(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) + throws IOException + { + return makeRequest(method, bucket, key, pathArgs, headers, null); + } + + private HttpURLConnection makeRequest(Method method, Bucket bucket) throws IOException { + return makeRequest(method, bucket, null); + } + + private HttpURLConnection makeRequest(Method method, Bucket bucket, Headers headers) throws IOException { + return makeRequest(method, bucket, null, headers); + } + + private HttpURLConnection makeRequest(Method method, Bucket bucket, Map<String, String> pathArgs, Headers headers) throws IOException + { + return makeRequest(method, bucket, "", pathArgs, headers); + } + + /** + * Make a new HttpURLConnection. + * @param method The HTTP method to use (GET, PUT, DELETE) + * @param bucketNamePattern The bucket name this request affects + * @param key The key this request is for, not encoded + * @param pathArgs parameters if any to be sent along this request + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + * @param object The S3Object that is to be written (can be null). + */ + private HttpURLConnection makeRequest(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers, + S3Object object) + throws IOException + { + String enckey = UrlEncoder.encode(key); + // build the domain based on the calling format + URL url = this.callingFormat.getURL(this.isSecure, server, this.port, bucket, enckey, pathArgs); + + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setRequestMethod(method.name()); + + // subdomain-style urls may encounter http redirects. + // Ensure that redirects are supported. + if (!connection.getInstanceFollowRedirects() + && callingFormat.supportsLocatedBuckets()) + throw new RuntimeException("HTTP redirect support required."); + + addHeaders(connection, headers); + if (object != null) + addMetadataHeaders(connection, object.getMetadata()); + addAuthHeader(connection, method, bucket, enckey, pathArgs); + + return connection; + } + + /** + * Add the given headers to the HttpURLConnection. + * @param connection The HttpURLConnection to which the headers will be added. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + */ + private void addHeaders(HttpURLConnection connection, Headers headers) { + addHeaders(connection, headers, ""); + } + + /** + * Add the given metadata fields to the HttpURLConnection. + * @param connection The HttpURLConnection to which the headers will be added. + * @param metadata A Map of String to List of Strings representing the s3 + * metadata for this resource. + */ + private void addMetadataHeaders(HttpURLConnection connection, Headers metadata) { + addHeaders(connection, metadata, Headers.METADATA_PREFIX); + } + + /** + * Add the given headers to the HttpURLConnection with a prefix before the keys. + * @param connection The HttpURLConnection to which the headers will be added. + * @param headers A Map of String to List of Strings representing the http + * headers to pass (can be null). + * @param prefix The string to prepend to each key before adding it to the connection. + */ + private void addHeaders(HttpURLConnection connection, Headers headers, String prefix) { + if (headers != null) { + for (Map.Entry<String, List<String>> me : headers.getHeaders().entrySet()) { + String key = me.getKey(); + for (String value : me.getValue()) { + connection.addRequestProperty(prefix + key, value); + } + } + } + } + + /** + * Add the appropriate Authorization header to the HttpURLConnection. + * @param connection The HttpURLConnection to which the header will be added. + * @param method The HTTP method to use (GET, PUT, DELETE) + * @param bucket the bucket name this request is for + * @param key the key this request is for (not URL encoded) + * @param pathArgs path arguments which are part of this request + */ + private void addAuthHeader(HttpURLConnection connection, Method method, Bucket bucket, String enckey, Map<String, String> pathArgs) { + if (connection.getRequestProperty("Date") == null) { + connection.setRequestProperty("Date", httpDate()); + } + if (connection.getRequestProperty("Content-Type") == null) { + connection.setRequestProperty("Content-Type", ""); + } + + Headers prop = new Headers(connection.getRequestProperties()); + String canonicalString = CanonicalString.make(method, bucket, enckey, pathArgs, prop); + String encodedCanonical = CanonicalString.encode(this.awsSecretAccessKey, canonicalString, false); + connection.setRequestProperty("Authorization", + "AWS " + this.awsAccessKeyId + ":" + encodedCanonical); + } + + /** + * Generate an rfc822 date for use in the Date HTTP header. + */ + private static String httpDate() { + final String DateFormat = "EEE, dd MMM yyyy HH:mm:ss "; + SimpleDateFormat format = new SimpleDateFormat( DateFormat, Locale.US ); + format.setTimeZone( TimeZone.getTimeZone( "UTC" ) ); + return format.format( new Date() ) + "GMT"; + } + + /** + * Returns a debug string. + */ + @Override + public String toString() { + return "Connection id=" + awsAccessKeyId + + " isSecure=" + isSecure + " server=" + server + + " port=" + port + " format=" + callingFormat; + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Entry.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Entry.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Entry.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,124 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.util.Date; + +/** + * A structure representing a single object stored in S3. + */ +public class Entry { + + private String key; + + private Date lastModified; + + private String eTag; + + private long size; + + private String storageClass; + + private Owner owner; + + /** + * Constructs a new Entry. + */ + Entry() { + } + + /** + * Sets lastModified. + */ + void setLastModified(Date lastModified) { + this.lastModified = lastModified; + } + + /** + * The date at which the object was last modified. + */ + public Date getLastModified() { + return (Date) lastModified.clone(); + } + + public String toString() { + return getKey(); + } + + /** + * Sets key. + */ + void setKey(String key) { + this.key = key; + } + + /** + * Returns the name of the object + */ + public String getKey() { + return key; + } + + /** + * Sets eTag. + */ + void setETag(String eTag) { + this.eTag = eTag; + } + + /** + * Returns the object's ETag, which can be used for conditional GETs. + */ + public String getETag() { + return eTag; + } + + /** + * Sets owner. + */ + void setOwner(Owner owner) { + this.owner = owner; + } + + /** + * Returns the object's owner + */ + public Owner getOwner() { + return owner; + } + + /** + * Sets size. + */ + void setSize(long size) { + this.size = size; + } + + /** + * Returns the size of the object in bytes. + */ + public long getSize() { + return size; + } + + /** + * Sets storageClass. + */ + void setStorageClass(String storageClass) { + this.storageClass = storageClass; + } + + /** + * The object's storage class + */ + public String getStorageClass() { + return storageClass; + } +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/GetResponse.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,66 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; + +/** + * A Response object returned from AWSAuthConnection.get(). Exposes the + * attribute object, which represents the retrieved object. + */ +public class GetResponse extends Response { + + private S3Object object; + + /** + * Pulls a representation of an S3Object out of the HttpURLConnection + * response. + */ + public GetResponse(HttpURLConnection connection) throws IOException { + super(connection); + if (connection.getResponseCode() < 400) { + Headers metadata = Headers.extractMetadata(connection); + int len = connection.getContentLength(); + byte[] body = slurpInputStream(connection.getInputStream(), len); + object = new S3Object(body, metadata); + } + } + + /** + * Read the input stream and dump it all into a big byte array + */ + private static byte[] slurpInputStream(InputStream stream, int len) throws IOException { + + final int chunkSize = 4 * 1024; + byte[] buf = new byte[chunkSize]; + ByteArrayOutputStream byteStream; + if (len > 0) { + byteStream = new ByteArrayOutputStream(len); + } else { + byteStream = new ByteArrayOutputStream(chunkSize); + } + + int count; + while ((count = stream.read(buf)) != -1) + byteStream.write(buf, 0, count); + + return byteStream.toByteArray(); + } + + /** + * Returns the S3 object. + */ + public S3Object getObject() { + return object; + } +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Headers.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,114 @@ +package com.amazon.s3; + +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * HTTP header wrapper. + * + * @author Elias Ross + */ +public class Headers { + + public static final String METADATA_PREFIX = "x-amz-meta-"; + + private Map<String, List<String>> headers; + + /** + * Constructs a new Headers object. + */ + public Headers(Map<String, List<String>> headers) { + this.headers = headers; + } + + /** + * Copy-constructs new Headers. + */ + public Headers(Headers headers) { + this(); + this.headers.putAll(headers.headers); + } + + /** + * Constructs a new Headers object. + */ + public Headers() { + this(new HashMap<String, List<String>>()); + } + + /** + * Adds a header. + */ + public void put(String header, String value) { + if (headers.containsKey(header)) { + headers.get(header).add(value); + } else { + headers.put(header, new ArrayList<String>(Collections.singletonList(value))); + } + } + + /** + * Returns headers. + */ + public Map<String, List<String>> getHeaders() { + return headers; + } + + /** + * Returns a list of values. + */ + public List<String> getValues(String header) { + return headers.get(header); + } + + /** + * Returns the number of headers. + */ + public int size() { + return headers.size(); + } + + /** + * Examines the response's header fields and returns a Map from String to + * List of Strings representing the object's metadata. + */ + public static Headers extractMetadata(HttpURLConnection connection) { + TreeMap<String, List<String>> metadata = new TreeMap<String, List<String>>(); + Map<String, List<String>> headers = connection.getHeaderFields(); + for (Map.Entry<String, List<String>> me : headers.entrySet()) { + String key = me.getKey(); + if (key == null) + continue; + if (key.startsWith(METADATA_PREFIX)) { + metadata.put(key.substring(METADATA_PREFIX.length()), me.getValue()); + } + } + + return new Headers(metadata); + } + + /** + * Returns new headers with metadata. + */ + public Headers mergeMetadata(Headers metadata) { + if (metadata == null) + return this; + Map<String, List<String>> merged = new TreeMap<String, List<String>>(this.headers); + for (Map.Entry<String, List<String>> me : metadata.headers.entrySet()) { + String key = me.getKey(); + String metadataKey = METADATA_PREFIX + key; + if (merged.containsKey(metadataKey)) { + merged.get(metadataKey).addAll(me.getValue()); + } else { + merged.put(metadataKey, me.getValue()); + } + } + return new Headers(merged); + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/ListAllBucketsResponse.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,98 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.TimeZone; + +import org.xml.sax.Attributes; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Response for listing all buckets. + */ +public class ListAllBucketsResponse extends Response { + + private List<Bucket> entries; + + public ListAllBucketsResponse(HttpURLConnection connection) throws IOException { + super(connection); + if (success()) { + entries = new ArrayList<Bucket>(); + parse(new ListAllMyBucketsHandler()); + } + } + + /** + * Returns the bucket entries, unmodifiable. + */ + public List<Bucket> getEntries() { + return Collections.unmodifiableList(entries); + } + + class ListAllMyBucketsHandler extends DefaultHandler { + + private Bucket currBucket = null; + private StringBuilder currText = null; + private SimpleDateFormat iso8601Parser = null; + + public ListAllMyBucketsHandler() { + super(); + this.iso8601Parser = new SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + this.iso8601Parser.setTimeZone(TimeZone.getTimeZone("UTC")); + this.currText = new StringBuilder(); + } + + public void startDocument() { + // ignore + } + + public void endDocument() { + // ignore + } + + public void startElement(String uri, String name, String qName, + Attributes attrs) { + if (name.equals("Bucket")) { + this.currBucket = new Bucket(); + } + } + + public void endElement(String uri, String name, String qName) { + if (name.equals("Bucket")) { + entries.add(this.currBucket); + } else if (name.equals("Name")) { + this.currBucket.setName(this.currText.toString()); + } else if (name.equals("CreationDate")) { + try { + this.currBucket.setCreationDate(this.iso8601Parser + .parse(this.currText.toString())); + } catch (ParseException e) { + throw new RuntimeException( + "Unexpected date format in list bucket output", e); + } + } + this.currText = new StringBuilder(); + } + + public void characters(char ch[], int start, int length) { + this.currText.append(ch, start, length); + } + + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/ListResponse.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,236 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; + +import org.xml.sax.Attributes; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Returned by {@link Connection#list}. + */ +public class ListResponse extends Response { + + private String name = null; + + private String prefix = null; + + private String marker = null; + + private String delimiter = null; + + private int maxKeys = 0; + + private boolean isTruncated = false; + + private String nextMarker = null; + + private List<Entry> entries = null; + + private List<CommonPrefixEntry> commonPrefixEntries = null; + + public ListResponse(HttpURLConnection connection) throws IOException { + super(connection); + if (success()) { + parse(new ListBucketHandler()); + } + } + + /** + * A List of CommonPrefixEntry objects representing the common prefixes of the + * keys that matched up to the delimiter. Null if the request fails. + */ + public List<CommonPrefixEntry> getCommonPrefixEntries() { + return commonPrefixEntries; + } + + /** + * Returns delimiter. + */ + public String getDelimiter() { + return delimiter; + } + + /** + * A List of ListEntry objects representing the objects in the given bucket. + */ + public List<Entry> getEntries() { + return entries; + } + + /** + * Indicates if there are more results to the list. True if the current + * list results have been truncated. false if request fails. + */ + public boolean isTruncated() { + return isTruncated; + } + + public class CommonPrefixEntry { + + /** + * The prefix common to the delimited keys it represents + */ + private String prefix; + + CommonPrefixEntry() { + } + + /** + * Returns prefix. + */ + public String getPrefix() { + return prefix; + } + } + + class ListBucketHandler extends DefaultHandler { + + private boolean isEchoedPrefix = false; + private Entry keyEntry = null; + private CommonPrefixEntry commonPrefixEntry = null; + private StringBuilder currText = null; + private SimpleDateFormat iso8601Parser = null; + + public ListBucketHandler() { + super(); + entries = new ArrayList<Entry>(); + commonPrefixEntries = new ArrayList<CommonPrefixEntry>(); + this.iso8601Parser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + this.iso8601Parser.setTimeZone(TimeZone.getTimeZone("UTC")); + this.currText = new StringBuilder(); + } + + public void startDocument() { + this.isEchoedPrefix = true; + } + + public void endDocument() { + // ignore + } + + public void startElement(String uri, String name, String qName, Attributes attrs) { + if (name.equals("Contents")) { + this.keyEntry = new Entry(); + } else if (name.equals("Owner")) { + this.keyEntry.setOwner(new Owner()); + } else if (name.equals("CommonPrefixes")){ + this.commonPrefixEntry = new CommonPrefixEntry(); + } + } + + public void endElement(String uri, String name, String qName) { + if (name.equals("Name")) { + name = this.currText.toString(); + } + // this prefix is the one we echo back from the request + else if (name.equals("Prefix") && this.isEchoedPrefix) { + prefix = this.currText.toString(); + this.isEchoedPrefix = false; + } else if (name.equals("Marker")) { + marker = this.currText.toString(); + } else if (name.equals("MaxKeys")) { + maxKeys = Integer.parseInt(this.currText.toString()); + } else if (name.equals("Delimiter")) { + delimiter = this.currText.toString(); + } else if (name.equals("IsTruncated")) { + isTruncated = Boolean.valueOf(this.currText.toString()); + } else if (name.equals("NextMarker")) { + nextMarker = this.currText.toString(); + } else if (name.equals("Contents")) { + entries.add(this.keyEntry); + } else if (name.equals("Key")) { + this.keyEntry.setKey(this.currText.toString()); + } else if (name.equals("LastModified")) { + try { + this.keyEntry.setLastModified(this.iso8601Parser.parse(this.currText.toString())); + } catch (ParseException e) { + throw new RuntimeException("Unexpected date format in list bucket output", e); + } + } else if (name.equals("ETag")) { + this.keyEntry.setETag(this.currText.toString()); + } else if (name.equals("Size")) { + this.keyEntry.setSize(Long.parseLong(this.currText.toString())); + } else if (name.equals("StorageClass")) { + this.keyEntry.setStorageClass(this.currText.toString()); + } else if (name.equals("ID")) { + this.keyEntry.getOwner().setId(this.currText.toString()); + } else if (name.equals("DisplayName")) { + this.keyEntry.getOwner().setDisplayName(this.currText.toString()); + } else if (name.equals("CommonPrefixes")) { + commonPrefixEntries.add(this.commonPrefixEntry); + } + // this is the common prefix for keys that match up to the delimiter + else if (name.equals("Prefix")) { + this.commonPrefixEntry.prefix = this.currText.toString(); + } + if(this.currText.length() != 0) + this.currText = new StringBuilder(); + } + + public void characters(char ch[], int start, int length) { + this.currText.append(ch, start, length); + } + + } + + /** + * The name of the bucket being listed. Null if request fails. + */ + public String getName() { + return this.name; + } + + /** + * The prefix echoed back from the request. Null if request fails. + */ + public String getPrefix() { + return this.prefix; + } + + /** + * The marker echoed back from the request. Null if request fails. + */ + public String getMarker() { + return this.marker; + } + + /** + * The maxKeys echoed back from the request if specified. 0 if request fails. + */ + public int getMaxKeys(){ + return this.maxKeys; + } + + /** + * The delimiter echoed back from the request. Null if not specified in + * the request, or if it fails. + */ + public boolean getIsTruncated() { + return this.isTruncated; + } + + /** + * Indicates what to use as a marker for subsequent list requests in the event + * that the results are truncated. Present only when a delimiter is specified. + * Null if request fails. + */ + public String getNextMarker() { + return this.nextMarker; + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/LocationResponse.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,74 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.IOException; +import java.net.HttpURLConnection; + +import org.xml.sax.Attributes; +import org.xml.sax.helpers.DefaultHandler; + +/** + * A Response object returned from AWSAuthConnection.getBucketLocation(). + * Parses the response XML and exposes the location constraint + * via the {@link #getLocation()} method. + */ +public class LocationResponse extends Response { + + private String location; + + /** + * Parse the response to a ?location query. + */ + public LocationResponse(HttpURLConnection connection) throws IOException { + super(connection); + if (success()) { + parse(new LocationResponseHandler()); + } + } + + /** + * Report the location-constraint for a bucket. + * A value of null indicates an error; + * the empty string indicates no constraint; + * and any other value is an actual location constraint value. + */ + public String getLocation() { + return location; + } + + /** + * Helper class to parse LocationConstraint response XML + */ + class LocationResponseHandler extends DefaultHandler { + private StringBuilder currText = null; + + public void startDocument() { + } + + public void startElement(String uri, String name, String qName, Attributes attrs) { + if (name.equals("LocationConstraint")) { + this.currText = new StringBuilder(); + } + } + + public void endElement(String uri, String name, String qName) { + if (name.equals("LocationConstraint")) { + location = this.currText.toString(); + this.currText = null; + } + } + + public void characters(char ch[], int start, int length) { + if (currText != null) + this.currText.append(ch, start, length); + } + } +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Method.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,10 @@ +package com.amazon.s3; + +/** + * HTTP Methods. + * + * @author Elias Ross + */ +public enum Method { + GET, PUT, DELETE, HEAD +} \ No newline at end of file Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Owner.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Owner.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Owner.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,59 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +/** + * A structure representing the owner of an object, used as a part of ListEntry. + */ +public class Owner { + + private String id; + private String displayName; + + Owner() { + } + + /** + * Sets displayName. + */ + void setDisplayName(String displayName) { + this.displayName = displayName; + } + + /** + * Returns displayName. + */ + public String getDisplayName() { + return displayName; + } + + /** + * Sets id. + */ + void setId(String id) { + this.id = id; + } + + /** + * Returns id. + */ + public String getId() { + return id; + } + + /** + * Returns a debug string. + */ + @Override + public String toString() + { + return "Owner id=" + id + " displayName=" + displayName; + } +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/QueryGenerator.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,240 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.net.MalformedURLException; +import java.net.URL; +import java.security.Key; +import java.util.HashMap; +import java.util.Map; + +/** + * Generates URL Query Strings that can be used to perform operations. + * These parameters include an expiration date, so that + * if you hand them off to someone else, they will only work for a limited amount of time. + */ +public class QueryGenerator { + + private String awsAccessKeyId; + private Key awsSecretAccessKey; + private boolean isSecure; + private String server; + private int port; + private CallingFormat callingFormat; + + private Long expiresIn = null; + private Long expires = null; + + // by default, expire in 1 minute. + private static final Long DEFAULT_EXPIRES_IN = new Long(60 * 1000); + + public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey) { + this(awsAccessKeyId, awsSecretAccessKey, true); + } + + public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey, + boolean isSecure) + { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, Connection.DEFAULT_HOST); + } + + public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey, + boolean isSecure, String server) + { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, + isSecure ? Connection.SECURE_PORT : Connection.INSECURE_PORT); + } + + public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey, + boolean isSecure, String server, int port) + { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, + port, CallingFormat.getSubdomainCallingFormat()); + } + + public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey, + boolean isSecure, String server, CallingFormat callingFormat) + { + this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, + isSecure ? Connection.SECURE_PORT : Connection.INSECURE_PORT, + callingFormat); + } + + public QueryGenerator(String awsAccessKeyId, String awsSecretAccessKey, + boolean isSecure, String server, int port, CallingFormat callingFormat) + { + this.awsAccessKeyId = awsAccessKeyId; + this.awsSecretAccessKey = CanonicalString.key(awsSecretAccessKey); + this.isSecure = isSecure; + this.server = server; + this.port = port; + this.callingFormat = callingFormat; + + this.expiresIn = DEFAULT_EXPIRES_IN; + this.expires = null; + } + + + public void setCallingFormat(CallingFormat format) { + this.callingFormat = format; + } + + public void setExpires(long millisSinceEpoch) { + expires = new Long(millisSinceEpoch); + expiresIn = null; + } + + public void setExpiresIn(long millis) { + expiresIn = new Long(millis); + expires = null; + } + + public URL create(Bucket bucket, Headers headers) + { + // validate bucket name + if (!bucket.validateName(callingFormat)) + throw new IllegalArgumentException("Invalid Bucket Name: "+bucket); + + return generateURL(Method.PUT, bucket, "", headers); + } + + public URL list(Bucket bucket, String prefix, String marker, + Integer maxKeys, Headers headers){ + return list(bucket, prefix, marker, maxKeys, null, headers); + } + + public URL list(Bucket bucket, String prefix, String marker, + Integer maxKeys, String delimiter, Headers headers) + { + return generateURL(Method.GET, bucket, headers); + } + + public URL delete(Bucket bucket, Headers headers) + { + return generateURL(Method.DELETE, bucket, headers); + } + + public URL put(Bucket bucket, String key, S3Object object, Headers headers) { + Headers metadata = null; + if (object != null) { + metadata = object.getMetadata(); + } + + return generateURL(Method.PUT, bucket, UrlEncoder.encode(key), headers.mergeMetadata(metadata)); + } + + public URL get(Bucket bucket, String key, Headers headers) + { + return generateURL(Method.GET, bucket, UrlEncoder.encode(key), headers); + } + + public URL delete(Bucket bucket, String key, Headers headers) + { + return generateURL(Method.DELETE, bucket, UrlEncoder.encode(key), headers); + } + + public URL getBucketLogging(Bucket bucket, Headers headers) { + Map<String, String> pathArgs = new HashMap<String, String>(); + pathArgs.put("logging", null); + return generateURL(Method.GET, bucket, "", pathArgs, headers); + } + + public URL putBucketLogging(Bucket bucket, String loggingXMLDoc, Headers headers) { + Map<String, String> pathArgs = new HashMap<String, String>(); + pathArgs.put("logging", null); + return generateURL(Method.PUT, bucket, "", pathArgs, headers); + } + + public URL getACL(Bucket bucket, Headers headers) { + return getACL(bucket, "", headers); + } + + public URL getACL(Bucket bucket, String key, Headers headers) + { + Map<String, String> pathArgs = new HashMap<String, String>(); + pathArgs.put("acl", null); + return generateURL(Method.GET, bucket, UrlEncoder.encode(key), pathArgs, headers); + } + + public URL putACL(Bucket bucket, String aclXMLDoc, Headers headers) { + return putACL(bucket, "", aclXMLDoc, headers); + } + + public URL putACL(Bucket bucket, String key, String aclXMLDoc, Headers headers) + { + Map<String, String> pathArgs = new HashMap<String, String>(); + pathArgs.put("acl", null); + return generateURL(Method.PUT, bucket, UrlEncoder.encode(key), pathArgs, headers); + } + + public URL listAllBuckets(Headers headers) + { + return generateURL(Method.GET, headers); + } + + public URL listAllBuckets() { + return listAllBuckets(null); + } + + public String makeBareURL(Bucket bucket, String key) { + StringBuilder buffer = new StringBuilder(); + if (this.isSecure) { + buffer.append("https://"); + } else { + buffer.append("http://"); + } + buffer.append(this.server).append(":").append(this.port).append("/").append(bucket); + buffer.append("/").append(UrlEncoder.encode(key)); + + return buffer.toString(); + } + + @SuppressWarnings("unchecked") + private URL generateURL(Method method, Bucket bucket, String key, Headers headers) { + return generateURL(method, bucket, key, new HashMap(), headers); + } + + private URL generateURL(Method method, Bucket bucket, Headers headers) { + return generateURL(method, bucket, "", headers); + } + + private URL generateURL(Method method, Headers headers) { + return generateURL(method, null, headers); + } + + private URL generateURL(Method method, Bucket bucket, String key, Map<String, String> pathArgs, Headers headers) + { + long expires = 0L; + if (this.expiresIn != null) { + expires = System.currentTimeMillis() + this.expiresIn.longValue(); + } else if (this.expires != null) { + expires = this.expires.longValue(); + } else { + throw new RuntimeException("Illegal expires state"); + } + + // convert to seconds + expires /= 1000; + + String canonicalString = CanonicalString.make(method, bucket, key, pathArgs, headers, ""+expires); + String encodedCanonical = CanonicalString.encode(this.awsSecretAccessKey, canonicalString, true); + + pathArgs.put("Signature", encodedCanonical); + pathArgs.put("Expires", Long.toString(expires)); + pathArgs.put("AWSAccessKeyId", this.awsAccessKeyId); + + try { + return this.callingFormat.getURL(this.isSecure, server, port, bucket, key, pathArgs); + } catch (MalformedURLException e) { + throw new IllegalStateException("Unable to generate URL " + e); + } + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/Response.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,152 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.HashMap; +import java.util.Map; + +import org.xml.sax.ContentHandler; +import org.xml.sax.ErrorHandler; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.XMLReaderFactory; + +/** + * The parent class of all other Responses. <p/> Returns status codes that + * should be checked. + */ +public class Response { + + private HttpURLConnection connection; + + public Response(HttpURLConnection connection) throws IOException { + this.connection = connection; + } + + /** + * Returns the HTTP response code. + */ + public int getResponseCode() throws IOException { + return connection.getResponseCode(); + } + + /** + * Returns the HTTP response message. + */ + public String getResponseMessage() throws IOException { + return connection.getResponseMessage(); + } + + /** + * Returns an HTTP header field from the response. + */ + public String getHeaderField(String field) { + return connection.getHeaderField(field); + } + + /** + * Returns true if processing was success. + */ + public boolean success() { + try { + return connection.getResponseCode() < 400; + } catch (IOException e) { + return false; + } + } + + /** + * Returns a string representation. + */ + @Override + public String toString() { + try { + return "Response code=" + getResponseCode() + " msg='" + + getResponseMessage() + "'"; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static Map<String, String> paramsForListOptions(String prefix, + String marker, Integer maxKeys) { + return paramsForListOptions(prefix, marker, maxKeys, null); + } + + static Map<String, String> paramsForListOptions(String prefix, + String marker, Integer maxKeys, String delimiter) { + + Map<String, String> argParams = new HashMap<String, String>(); + // these three params must be url encoded + if (prefix != null) + argParams.put("prefix", UrlEncoder.encode(prefix)); + if (marker != null) + argParams.put("marker", UrlEncoder.encode(marker)); + if (delimiter != null) + argParams.put("delimiter", UrlEncoder.encode(delimiter)); + + if (maxKeys != null) + argParams.put("max-keys", Integer.toString(maxKeys.intValue())); + + return argParams; + + } + + /** + * Parses the response. + */ + protected void parse(ContentHandler handler) throws IOException { + try { + XMLReader xr = createXMLReader(); + xr.setContentHandler(handler); + ErrorHandler eh = new ErrorHandler() { + + public void error(SAXParseException e) throws SAXException { + throw e; + } + + public void fatalError(SAXParseException e) throws SAXException { + throw e; + } + + public void warning(SAXParseException e) throws SAXException { + throw e; + } + + }; + xr.setErrorHandler(eh); + xr.parse(new InputSource(connection.getInputStream())); + } catch (SAXException e) { + throw new RuntimeException("Unexpected error parsing ListBucket xml", e); + } + } + + static XMLReader createXMLReader() { + try { + return XMLReaderFactory.createXMLReader(); + } catch (SAXException e) { + // oops, lets try doing this (needed in 1.4) + System.setProperty("org.xml.sax.driver", + "org.apache.crimson.parser.XMLReaderImpl"); + } + try { + // try once more + return XMLReaderFactory.createXMLReader(); + } catch (SAXException e) { + throw new RuntimeException( + "Couldn't initialize a sax driver for the XMLReader"); + } + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/S3Object.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,64 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + + + +/** + * A representation of a single object stored in S3. + */ +public class S3Object { + + /** + * Data itself. + */ + private byte[] data; + + /** + * Meta-data headers. + */ + private Headers metadata; + + /** + * Constructs a new S3Object. + * @param data + * @param metadata + */ + public S3Object(byte[] data, Headers metadata) { + this.data = data; + this.metadata = metadata; + } + + /** + * Returns data. + */ + public byte[] getData() { + return data; + } + + /** + * Returns metadata, unmodifiable. + */ + public Headers getMetadata() { + return metadata; + } + + /** + * Returns a debug string. + */ + @Override + public String toString() + { + return "S3Object[ " + + "data=" + this.data.length + + "metadata=" + this.metadata + "]"; + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/s3/UrlEncoder.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/s3/UrlEncoder.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/s3/UrlEncoder.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,28 @@ +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + +package com.amazon.s3; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + + + +public class UrlEncoder { + + static String encode(String unencoded) { + try { + return URLEncoder.encode(unencoded, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // should never happen + throw new RuntimeException("Could not url encode to UTF-8", e); + } + } + +} Added: amazon-s3/trunk/src/main/java/com/amazon/thirdparty/Base64.java =================================================================== --- amazon-s3/trunk/src/main/java/com/amazon/thirdparty/Base64.java (rev 0) +++ amazon-s3/trunk/src/main/java/com/amazon/thirdparty/Base64.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,1459 @@ +// +// NOTE: The following source code is the iHarder.net public domain +// Base64 library and is provided here as a convenience. For updates, +// problems, questions, etc. regarding this code, please visit: +// http://iharder.sourceforge.net/current/java/base64/ +// + +package com.amazon.thirdparty; + + +/** + * Encodes and decodes to and from Base64 notation. + * + * <p> + * Change Log: + * </p> + * <ul> + * <li>v2.1 - Cleaned up javadoc comments and unused variables and methods. Added + * some convenience methods for reading and writing to and from files.</li> + * <li>v2.0.2 - Now specifies UTF-8 encoding in places where the code fails on systems + * with other encodings (like EBCDIC).</li> + * <li>v2.0.1 - Fixed an error when decoding a single byte, that is, when the + * encoded data was a single byte.</li> + * <li>v2.0 - I got rid of methods that used booleans to set options. + * Now everything is more consolidated and cleaner. The code now detects + * when data that's being decoded is gzip-compressed and will decompress it + * automatically. Generally things are cleaner. You'll probably have to + * change some method calls that you were making to support the new + * options format (<tt>int</tt>s that you "OR" together).</li> + * <li>v1.5.1 - Fixed bug when decompressing and decoding to a + * byte[] using <tt>decode( String s, boolean gzipCompressed )</tt>. + * Added the ability to "suspend" encoding in the Output Stream so + * you can turn on and off the encoding if you need to embed base64 + * data in an otherwise "normal" stream (like an XML file).</li> + * <li>v1.5 - Output stream pases on flush() command but doesn't do anything itself. + * This helps when using GZIP streams. + * Added the ability to GZip-compress objects before encoding them.</li> + * <li>v1.4 - Added helper methods to read/write files.</li> + * <li>v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.</li> + * <li>v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream + * where last buffer being read, if not completely full, was not returned.</li> + * <li>v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.</li> + * <li>v1.3.3 - Fixed I/O streams which were totally messed up.</li> + * </ul> + * + * <p> + * I am placing this code in the Public Domain. Do with it as you will. + * This software comes with no guarantees or warranties but with + * plenty of well-wishing instead! + * Please visit <a href="http://iharder.net/base64">http://iharder.net/base64&l... + * periodically to check for updates or to contribute improvements. + * </p> + * + * @author Robert Harder + * @author rob(a)iharder.net + * @version 2.1 + */ +public class Base64 +{ + +/* ******** P U B L I C F I E L D S ******** */ + + + /** No options specified. Value is zero. */ + public final static int NO_OPTIONS = 0; + + /** Specify encoding. */ + public final static int ENCODE = 1; + + + /** Specify decoding. */ + public final static int DECODE = 0; + + + /** Specify that data should be gzip-compressed. */ + public final static int GZIP = 2; + + + /** Don't break lines when encoding (violates strict Base64 specification) */ + public final static int DONT_BREAK_LINES = 8; + + +/* ******** P R I V A T E F I E L D S ******** */ + + + /** Maximum line length (76) of Base64 output. */ + private final static int MAX_LINE_LENGTH = 76; + + + /** The equals sign (=) as a byte. */ + private final static byte EQUALS_SIGN = (byte)'='; + + + /** The new line character (\n) as a byte. */ + private final static byte NEW_LINE = (byte)'\n'; + + + /** Preferred encoding. */ + private final static String PREFERRED_ENCODING = "UTF-8"; + + + /** The 64 valid Base64 values. */ + private final static byte[] ALPHABET; + private final static byte[] _NATIVE_ALPHABET = /* May be something funny like EBCDIC */ + { + (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G', + (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N', + (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', + (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z', + (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g', + (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n', + (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', + (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z', + (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', + (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/' + }; + + /** Determine which ALPHABET to use. */ + static + { + byte[] __bytes; + try + { + __bytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".getBytes( PREFERRED_ENCODING ); + } // end try + catch (java.io.UnsupportedEncodingException use) + { + __bytes = _NATIVE_ALPHABET; // Fall back to native encoding + } // end catch + ALPHABET = __bytes; + } // end static + + + /** + * Translates a Base64 value to either its 6-bit reconstruction value + * or a negative number indicating some other meaning. + **/ + private final static byte[] DECODABET = + { + -9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 0 - 8 + -5,-5, // Whitespace: Tab and Linefeed + -9,-9, // Decimal 11 - 12 + -5, // Whitespace: Carriage Return + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 14 - 26 + -9,-9,-9,-9,-9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 33 - 42 + 62, // Plus sign at decimal 43 + -9,-9,-9, // Decimal 44 - 46 + 63, // Slash at decimal 47 + 52,53,54,55,56,57,58,59,60,61, // Numbers zero through nine + -9,-9,-9, // Decimal 58 - 60 + -1, // Equals sign at decimal 61 + -9,-9,-9, // Decimal 62 - 64 + 0,1,2,3,4,5,6,7,8,9,10,11,12,13, // Letters 'A' through 'N' + 14,15,16,17,18,19,20,21,22,23,24,25, // Letters 'O' through 'Z' + -9,-9,-9,-9,-9,-9, // Decimal 91 - 96 + 26,27,28,29,30,31,32,33,34,35,36,37,38, // Letters 'a' through 'm' + 39,40,41,42,43,44,45,46,47,48,49,50,51, // Letters 'n' through 'z' + -9,-9,-9,-9 // Decimal 123 - 126 + /*,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 127 - 139 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 140 - 152 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 153 - 165 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 166 - 178 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 179 - 191 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 192 - 204 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 205 - 217 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 218 - 230 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 231 - 243 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9 // Decimal 244 - 255 */ + }; + + // I think I end up not using the BAD_ENCODING indicator. + //private final static byte BAD_ENCODING = -9; // Indicates error in encoding + private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding + private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding + + + /** Defeats instantiation. */ + private Base64(){} + + + +/* ******** E N C O D I N G M E T H O D S ******** */ + + + /** + * Encodes up to the first three bytes of array <var>threeBytes</var> + * and returns a four-byte array in Base64 notation. + * The actual number of significant bytes in your array is + * given by <var>numSigBytes</var>. + * The array <var>threeBytes</var> needs only be as big as + * <var>numSigBytes</var>. + * Code can reuse a byte array by passing a four-byte array as <var>b4</var>. + * + * @param b4 A reusable byte array to reduce array instantiation + * @param threeBytes the array to convert + * @param numSigBytes the number of significant bytes in your array + * @return four byte array in Base64 notation. + * @since 1.5.1 + */ + private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes ) + { + encode3to4( threeBytes, 0, numSigBytes, b4, 0 ); + return b4; + } // end encode3to4 + + + /** + * Encodes up to three bytes of the array <var>source</var> + * and writes the resulting four Base64 bytes to <var>destination</var>. + * The source and destination arrays can be manipulated + * anywhere along their length by specifying + * <var>srcOffset</var> and <var>destOffset</var>. + * This method does not check to make sure your arrays + * are large enough to accomodate <var>srcOffset</var> + 3 for + * the <var>source</var> array or <var>destOffset</var> + 4 for + * the <var>destination</var> array. + * The actual number of significant bytes in your array is + * given by <var>numSigBytes</var>. + * + * @param source the array to convert + * @param srcOffset the index where conversion begins + * @param numSigBytes the number of significant bytes in your array + * @param destination the array to hold the conversion + * @param destOffset the index where output will be put + * @return the <var>destination</var> array + * @since 1.3 + */ + private static byte[] encode3to4( + byte[] source, int srcOffset, int numSigBytes, + byte[] destination, int destOffset ) + { + // 1 2 3 + // 01234567890123456789012345678901 Bit position + // --------000000001111111122222222 Array position from threeBytes + // --------| || || || | Six bit groups to index ALPHABET + // >>18 >>12 >> 6 >> 0 Right shift necessary + // 0x3f 0x3f 0x3f Additional AND + + // Create buffer with zero-padding if there are only one or two + // significant bytes passed in the array. + // We have to shift left 24 in order to flush out the 1's that appear + // when Java treats a value as negative that is cast from a byte to an int. + int inBuff = ( numSigBytes > 0 ? ((source[ srcOffset ] << 24) >>> 8) : 0 ) + | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 ) + | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 ); + + switch( numSigBytes ) + { + case 3: + destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; + destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; + destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>> 6) & 0x3f ]; + destination[ destOffset + 3 ] = ALPHABET[ (inBuff ) & 0x3f ]; + return destination; + + case 2: + destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; + destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; + destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>> 6) & 0x3f ]; + destination[ destOffset + 3 ] = EQUALS_SIGN; + return destination; + + case 1: + destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; + destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; + destination[ destOffset + 2 ] = EQUALS_SIGN; + destination[ destOffset + 3 ] = EQUALS_SIGN; + return destination; + + default: + return destination; + } // end switch + } // end encode3to4 + + + + /** + * Serializes an object and returns the Base64-encoded + * version of that serialized object. If the object + * cannot be serialized or there is another error, + * the method will return <tt>null</tt>. + * The object is not GZip-compressed before being encoded. + * + * @param serializableObject The object to encode + * @return The Base64-encoded object + * @since 1.4 + */ + public static String encodeObject( java.io.Serializable serializableObject ) + { + return encodeObject( serializableObject, NO_OPTIONS ); + } // end encodeObject + + + + /** + * Serializes an object and returns the Base64-encoded + * version of that serialized object. If the object + * cannot be serialized or there is another error, + * the method will return <tt>null</tt>. + * <p> + * Valid options:<pre> + * GZIP: gzip-compresses object before encoding it. + * DONT_BREAK_LINES: don't break lines at 76 characters + * <i>Note: Technically, this makes your encoding non-compliant.</i> + * </pre> + * <p> + * Example: <code>encodeObject( myObj, Base64.GZIP )</code> or + * <p> + * Example: <code>encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES )</code> + * + * @param serializableObject The object to encode + * @param options Specified options + * @return The Base64-encoded object + * @see Base64#GZIP + * @see Base64#DONT_BREAK_LINES + * @since 2.0 + */ + public static String encodeObject( java.io.Serializable serializableObject, int options ) + { + // Streams + java.io.ByteArrayOutputStream baos = null; + java.io.OutputStream b64os = null; + java.io.ObjectOutputStream oos = null; + java.util.zip.GZIPOutputStream gzos = null; + + // Isolate options + int gzip = (options & GZIP); + int dontBreakLines = (options & DONT_BREAK_LINES); + + try + { + // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream + baos = new java.io.ByteArrayOutputStream(); + b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines ); + + // GZip? + if( gzip == GZIP ) + { + gzos = new java.util.zip.GZIPOutputStream( b64os ); + oos = new java.io.ObjectOutputStream( gzos ); + } // end if: gzip + else + oos = new java.io.ObjectOutputStream( b64os ); + + oos.writeObject( serializableObject ); + } // end try + catch( java.io.IOException e ) + { + e.printStackTrace(); + return null; + } // end catch + finally + { + try{ oos.close(); } catch( Exception e ){} + try{ gzos.close(); } catch( Exception e ){} + try{ b64os.close(); } catch( Exception e ){} + try{ baos.close(); } catch( Exception e ){} + } // end finally + + // Return value according to relevant encoding. + try + { + return new String( baos.toByteArray(), PREFERRED_ENCODING ); + } // end try + catch (java.io.UnsupportedEncodingException uue) + { + return new String( baos.toByteArray() ); + } // end catch + + } // end encode + + + + /** + * Encodes a byte array into Base64 notation. + * Does not GZip-compress data. + * + * @param source The data to convert + * @since 1.4 + */ + public static String encodeBytes( byte[] source ) + { + return encodeBytes( source, 0, source.length, NO_OPTIONS ); + } // end encodeBytes + + + + /** + * Encodes a byte array into Base64 notation. + * <p> + * Valid options:<pre> + * GZIP: gzip-compresses object before encoding it. + * DONT_BREAK_LINES: don't break lines at 76 characters + * <i>Note: Technically, this makes your encoding non-compliant.</i> + * </pre> + * <p> + * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or + * <p> + * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code> + * + * + * @param source The data to convert + * @param options Specified options + * @see Base64#GZIP + * @see Base64#DONT_BREAK_LINES + * @since 2.0 + */ + public static String encodeBytes( byte[] source, int options ) + { + return encodeBytes( source, 0, source.length, options ); + } // end encodeBytes + + + /** + * Encodes a byte array into Base64 notation. + * Does not GZip-compress data. + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @since 1.4 + */ + public static String encodeBytes( byte[] source, int off, int len ) + { + return encodeBytes( source, off, len, NO_OPTIONS ); + } // end encodeBytes + + + + /** + * Encodes a byte array into Base64 notation. + * <p> + * Valid options:<pre> + * GZIP: gzip-compresses object before encoding it. + * DONT_BREAK_LINES: don't break lines at 76 characters + * <i>Note: Technically, this makes your encoding non-compliant.</i> + * </pre> + * <p> + * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or + * <p> + * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code> + * + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @param options Specified options + * @see Base64#GZIP + * @see Base64#DONT_BREAK_LINES + * @since 2.0 + */ + public static String encodeBytes( byte[] source, int off, int len, int options ) + { + // Isolate options + int dontBreakLines = ( options & DONT_BREAK_LINES ); + int gzip = ( options & GZIP ); + + // Compress? + if( gzip == GZIP ) + { + java.io.ByteArrayOutputStream baos = null; + java.util.zip.GZIPOutputStream gzos = null; + Base64.OutputStream b64os = null; + + + try + { + // GZip -> Base64 -> ByteArray + baos = new java.io.ByteArrayOutputStream(); + b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines ); + gzos = new java.util.zip.GZIPOutputStream( b64os ); + + gzos.write( source, off, len ); + gzos.close(); + } // end try + catch( java.io.IOException e ) + { + e.printStackTrace(); + return null; + } // end catch + finally + { + try{ gzos.close(); } catch( Exception e ){} + try{ b64os.close(); } catch( Exception e ){} + try{ baos.close(); } catch( Exception e ){} + } // end finally + + // Return value according to relevant encoding. + try + { + return new String( baos.toByteArray(), PREFERRED_ENCODING ); + } // end try + catch (java.io.UnsupportedEncodingException uue) + { + return new String( baos.toByteArray() ); + } // end catch + } // end if: compress + + // Else, don't compress. Better not to use streams at all then. + else + { + // Convert option to boolean in way that code likes it. + boolean breakLines = dontBreakLines == 0; + + int len43 = len * 4 / 3; + byte[] outBuff = new byte[ ( len43 ) // Main 4:3 + + ( (len % 3) > 0 ? 4 : 0 ) // Account for padding + + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines + int d = 0; + int e = 0; + int len2 = len - 2; + int lineLength = 0; + for( ; d < len2; d+=3, e+=4 ) + { + encode3to4( source, d+off, 3, outBuff, e ); + + lineLength += 4; + if( breakLines && lineLength == MAX_LINE_LENGTH ) + { + outBuff[e+4] = NEW_LINE; + e++; + lineLength = 0; + } // end if: end of line + } // en dfor: each piece of array + + if( d < len ) + { + encode3to4( source, d+off, len - d, outBuff, e ); + e += 4; + } // end if: some padding needed + + + // Return value according to relevant encoding. + try + { + return new String( outBuff, 0, e, PREFERRED_ENCODING ); + } // end try + catch (java.io.UnsupportedEncodingException uue) + { + return new String( outBuff, 0, e ); + } // end catch + + } // end else: don't compress + + } // end encodeBytes + + + + + +/* ******** D E C O D I N G M E T H O D S ******** */ + + + /** + * Decodes four bytes from array <var>source</var> + * and writes the resulting bytes (up to three of them) + * to <var>destination</var>. + * The source and destination arrays can be manipulated + * anywhere along their length by specifying + * <var>srcOffset</var> and <var>destOffset</var>. + * This method does not check to make sure your arrays + * are large enough to accomodate <var>srcOffset</var> + 4 for + * the <var>source</var> array or <var>destOffset</var> + 3 for + * the <var>destination</var> array. + * This method returns the actual number of bytes that + * were converted from the Base64 encoding. + * + * + * @param source the array to convert + * @param srcOffset the index where conversion begins + * @param destination the array to hold the conversion + * @param destOffset the index where output will be put + * @return the number of decoded bytes converted + * @since 1.3 + */ + private static int decode4to3( byte[] source, int srcOffset, byte[] destination, int destOffset ) + { + // Example: Dk== + if( source[ srcOffset + 2] == EQUALS_SIGN ) + { + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 ); + int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) + | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 ); + + destination[ destOffset ] = (byte)( outBuff >>> 16 ); + return 1; + } + + // Example: DkL= + else if( source[ srcOffset + 3 ] == EQUALS_SIGN ) + { + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) + // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ); + int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) + | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 ) + | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) << 6 ); + + destination[ destOffset ] = (byte)( outBuff >>> 16 ); + destination[ destOffset + 1 ] = (byte)( outBuff >>> 8 ); + return 2; + } + + // Example: DkLE + else + { + try{ + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) + // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ) + // | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 ); + int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) + | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 ) + | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) << 6) + | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF ) ); + + + destination[ destOffset ] = (byte)( outBuff >> 16 ); + destination[ destOffset + 1 ] = (byte)( outBuff >> 8 ); + destination[ destOffset + 2 ] = (byte)( outBuff ); + + return 3; + }catch( Exception e){ + System.out.println(""+source[srcOffset]+ ": " + ( DECODABET[ source[ srcOffset ] ] ) ); + System.out.println(""+source[srcOffset+1]+ ": " + ( DECODABET[ source[ srcOffset + 1 ] ] ) ); + System.out.println(""+source[srcOffset+2]+ ": " + ( DECODABET[ source[ srcOffset + 2 ] ] ) ); + System.out.println(""+source[srcOffset+3]+ ": " + ( DECODABET[ source[ srcOffset + 3 ] ] ) ); + return -1; + } //e nd catch + } + } // end decodeToBytes + + + + + /** + * Very low-level access to decoding ASCII characters in + * the form of a byte array. Does not support automatically + * gunzipping or any other "fancy" features. + * + * @param source The Base64 encoded data + * @param off The offset of where to begin decoding + * @param len The length of characters to decode + * @return decoded data + * @since 1.3 + */ + public static byte[] decode( byte[] source, int off, int len ) + { + int len34 = len * 3 / 4; + byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output + int outBuffPosn = 0; + + byte[] b4 = new byte[4]; + int b4Posn = 0; + int i = 0; + byte sbiCrop = 0; + byte sbiDecode = 0; + for( i = off; i < off+len; i++ ) + { + sbiCrop = (byte)(source[i] & 0x7f); // Only the low seven bits + sbiDecode = DECODABET[ sbiCrop ]; + + if( sbiDecode >= WHITE_SPACE_ENC ) // White space, Equals sign or better + { + if( sbiDecode >= EQUALS_SIGN_ENC ) + { + b4[ b4Posn++ ] = sbiCrop; + if( b4Posn > 3 ) + { + outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn ); + b4Posn = 0; + + // If that was the equals sign, break out of 'for' loop + if( sbiCrop == EQUALS_SIGN ) + break; + } // end if: quartet built + + } // end if: equals sign or better + + } // end if: white space, equals sign or better + else + { + System.err.println( "Bad Base64 input character at " + i + ": " + source[i] + "(decimal)" ); + return null; + } // end else: + } // each input character + + byte[] out = new byte[ outBuffPosn ]; + System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); + return out; + } // end decode + + + + + /** + * Decodes data from Base64 notation, automatically + * detecting gzip-compressed data and decompressing it. + * + * @param s the string to decode + * @return the decoded data + * @since 1.4 + */ + public static byte[] decode( String s ) + { + byte[] bytes; + try + { + bytes = s.getBytes( PREFERRED_ENCODING ); + } // end try + catch( java.io.UnsupportedEncodingException uee ) + { + bytes = s.getBytes(); + } // end catch + //</change> + + // Decode + bytes = decode( bytes, 0, bytes.length ); + + + // Check to see if it's gzip-compressed + // GZIP Magic Two-Byte Number: 0x8b1f (35615) + if( bytes != null && bytes.length >= 4 ) + { + + int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00); + if( java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) + { + java.io.ByteArrayInputStream bais = null; + java.util.zip.GZIPInputStream gzis = null; + java.io.ByteArrayOutputStream baos = null; + byte[] buffer = new byte[2048]; + int length = 0; + + try + { + baos = new java.io.ByteArrayOutputStream(); + bais = new java.io.ByteArrayInputStream( bytes ); + gzis = new java.util.zip.GZIPInputStream( bais ); + + while( ( length = gzis.read( buffer ) ) >= 0 ) + { + baos.write(buffer,0,length); + } // end while: reading input + + // No error? Get new bytes. + bytes = baos.toByteArray(); + + } // end try + catch( java.io.IOException e ) + { + // Just return originally-decoded bytes + } // end catch + finally + { + try{ baos.close(); } catch( Exception e ){} + try{ gzis.close(); } catch( Exception e ){} + try{ bais.close(); } catch( Exception e ){} + } // end finally + + } // end if: gzipped + } // end if: bytes.length >= 2 + + return bytes; + } // end decode + + + + + /** + * Attempts to decode Base64 data and deserialize a Java + * Object within. Returns <tt>null</tt> if there was an error. + * + * @param encodedObject The Base64 data to decode + * @return The decoded and deserialized object + * @since 1.5 + */ + public static Object decodeToObject( String encodedObject ) + { + // Decode and gunzip if necessary + byte[] objBytes = decode( encodedObject ); + + java.io.ByteArrayInputStream bais = null; + java.io.ObjectInputStream ois = null; + Object obj = null; + + try + { + bais = new java.io.ByteArrayInputStream( objBytes ); + ois = new java.io.ObjectInputStream( bais ); + + obj = ois.readObject(); + } // end try + catch( java.io.IOException e ) + { + e.printStackTrace(); + obj = null; + } // end catch + catch( java.lang.ClassNotFoundException e ) + { + e.printStackTrace(); + obj = null; + } // end catch + finally + { + try{ bais.close(); } catch( Exception e ){} + try{ ois.close(); } catch( Exception e ){} + } // end finally + + return obj; + } // end decodeObject + + + + /** + * Convenience method for encoding data to a file. + * + * @param dataToEncode byte array of data to encode in base64 form + * @param filename Filename for saving encoded data + * @return <tt>true</tt> if successful, <tt>false</tt> otherwise + * + * @since 2.1 + */ + public static boolean encodeToFile( byte[] dataToEncode, String filename ) + { + boolean success = false; + Base64.OutputStream bos = null; + try + { + bos = new Base64.OutputStream( + new java.io.FileOutputStream( filename ), Base64.ENCODE ); + bos.write( dataToEncode ); + success = true; + } // end try + catch( java.io.IOException e ) + { + + success = false; + } // end catch: IOException + finally + { + try{ bos.close(); } catch( Exception e ){} + } // end finally + + return success; + } // end encodeToFile + + + /** + * Convenience method for decoding data to a file. + * + * @param dataToDecode Base64-encoded data as a string + * @param filename Filename for saving decoded data + * @return <tt>true</tt> if successful, <tt>false</tt> otherwise + * + * @since 2.1 + */ + public static boolean decodeToFile( String dataToDecode, String filename ) + { + boolean success = false; + Base64.OutputStream bos = null; + try + { + bos = new Base64.OutputStream( + new java.io.FileOutputStream( filename ), Base64.DECODE ); + bos.write( dataToDecode.getBytes( PREFERRED_ENCODING ) ); + success = true; + } // end try + catch( java.io.IOException e ) + { + success = false; + } // end catch: IOException + finally + { + try{ bos.close(); } catch( Exception e ){} + } // end finally + + return success; + } // end decodeToFile + + + + + /** + * Convenience method for reading a base64-encoded + * file and decoding it. + * + * @param filename Filename for reading encoded data + * @return decoded byte array or null if unsuccessful + * + * @since 2.1 + */ + public static byte[] decodeFromFile( String filename ) + { + byte[] decodedData = null; + Base64.InputStream bis = null; + try + { + // Set up some useful variables + java.io.File file = new java.io.File( filename ); + byte[] buffer = null; + int length = 0; + int numBytes = 0; + + // Check for size of file + if( file.length() > Integer.MAX_VALUE ) + { + System.err.println( "File is too big for this convenience method (" + file.length() + " bytes)." ); + return null; + } // end if: file too big for int index + buffer = new byte[ (int)file.length() ]; + + // Open a stream + bis = new Base64.InputStream( + new java.io.BufferedInputStream( + new java.io.FileInputStream( file ) ), Base64.DECODE ); + + // Read until done + while( ( numBytes = bis.read( buffer, length, 4096 ) ) >= 0 ) + length += numBytes; + + // Save in a variable to return + decodedData = new byte[ length ]; + System.arraycopy( buffer, 0, decodedData, 0, length ); + + } // end try + catch( java.io.IOException e ) + { + System.err.println( "Error decoding from file " + filename ); + } // end catch: IOException + finally + { + try{ bis.close(); } catch( Exception e) {} + } // end finally + + return decodedData; + } // end decodeFromFile + + + + /** + * Convenience method for reading a binary file + * and base64-encoding it. + * + * @param filename Filename for reading binary data + * @return base64-encoded string or null if unsuccessful + * + * @since 2.1 + */ + public static String encodeFromFile( String filename ) + { + String encodedData = null; + Base64.InputStream bis = null; + try + { + // Set up some useful variables + java.io.File file = new java.io.File( filename ); + byte[] buffer = new byte[ (int)(file.length() * 1.4) ]; + int length = 0; + int numBytes = 0; + + // Open a stream + bis = new Base64.InputStream( + new java.io.BufferedInputStream( + new java.io.FileInputStream( file ) ), Base64.ENCODE ); + + // Read until done + while( ( numBytes = bis.read( buffer, length, 4096 ) ) >= 0 ) + length += numBytes; + + // Save in a variable to return + encodedData = new String( buffer, 0, length, Base64.PREFERRED_ENCODING ); + + } // end try + catch( java.io.IOException e ) + { + System.err.println( "Error encoding from file " + filename ); + } // end catch: IOException + finally + { + try{ bis.close(); } catch( Exception e) {} + } // end finally + + return encodedData; + } // end encodeFromFile + + + + + /* ******** I N N E R C L A S S I N P U T S T R E A M ******** */ + + + + /** + * A {@link Base64.InputStream} will read data from another + * <tt>java.io.InputStream</tt>, given in the constructor, + * and encode/decode to/from Base64 notation on the fly. + * + * @see Base64 + * @since 1.3 + */ + public static class InputStream extends java.io.FilterInputStream + { + private boolean encode; // Encoding or decoding + private int position; // Current position in the buffer + private byte[] buffer; // Small buffer holding converted data + private int bufferLength; // Length of buffer (3 or 4) + private int numSigBytes; // Number of meaningful bytes in the buffer + private int lineLength; + private boolean breakLines; // Break lines at less than 80 characters + + + /** + * Constructs a {@link Base64.InputStream} in DECODE mode. + * + * @param in the <tt>java.io.InputStream</tt> from which to read data. + * @since 1.3 + */ + public InputStream( java.io.InputStream in ) + { + this( in, DECODE ); + } // end constructor + + + /** + * Constructs a {@link Base64.InputStream} in + * either ENCODE or DECODE mode. + * <p> + * Valid options:<pre> + * ENCODE or DECODE: Encode or Decode as data is read. + * DONT_BREAK_LINES: don't break lines at 76 characters + * (only meaningful when encoding) + * <i>Note: Technically, this makes your encoding non-compliant.</i> + * </pre> + * <p> + * Example: <code>new Base64.InputStream( in, Base64.DECODE )</code> + * + * + * @param in the <tt>java.io.InputStream</tt> from which to read data. + * @param options Specified options + * @see Base64#ENCODE + * @see Base64#DECODE + * @see Base64#DONT_BREAK_LINES + * @since 2.0 + */ + public InputStream( java.io.InputStream in, int options ) + { + super( in ); + this.breakLines = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES; + this.encode = (options & ENCODE) == ENCODE; + this.bufferLength = encode ? 4 : 3; + this.buffer = new byte[ bufferLength ]; + this.position = -1; + this.lineLength = 0; + } // end constructor + + /** + * Reads enough of the input stream to convert + * to/from Base64 and returns the next byte. + * + * @return next byte + * @since 1.3 + */ + public int read() throws java.io.IOException + { + // Do we need to get data? + if( position < 0 ) + { + if( encode ) + { + byte[] b3 = new byte[3]; + int numBinaryBytes = 0; + for( int i = 0; i < 3; i++ ) + { + try + { + int b = in.read(); + + // If end of stream, b is -1. + if( b >= 0 ) + { + b3[i] = (byte)b; + numBinaryBytes++; + } // end if: not end of stream + + } // end try: read + catch( java.io.IOException e ) + { + // Only a problem if we got no data at all. + if( i == 0 ) + throw e; + + } // end catch + } // end for: each needed input byte + + if( numBinaryBytes > 0 ) + { + encode3to4( b3, 0, numBinaryBytes, buffer, 0 ); + position = 0; + numSigBytes = 4; + } // end if: got data + else + { + return -1; + } // end else + } // end if: encoding + + // Else decoding + else + { + byte[] b4 = new byte[4]; + int i = 0; + for( i = 0; i < 4; i++ ) + { + // Read four "meaningful" bytes: + int b = 0; + do{ b = in.read(); } + while( b >= 0 && DECODABET[ b & 0x7f ] <= WHITE_SPACE_ENC ); + + if( b < 0 ) + break; // Reads a -1 if end of stream + + b4[i] = (byte)b; + } // end for: each needed input byte + + if( i == 4 ) + { + numSigBytes = decode4to3( b4, 0, buffer, 0 ); + position = 0; + } // end if: got four characters + else if( i == 0 ){ + return -1; + } // end else if: also padded correctly + else + { + // Must have broken out from above. + throw new java.io.IOException( "Improperly padded Base64 input." ); + } // end + + } // end else: decode + } // end else: get data + + // Got data? + if( position >= 0 ) + { + // End of relevant data? + if( /*!encode &&*/ position >= numSigBytes ) + return -1; + + if( encode && breakLines && lineLength >= MAX_LINE_LENGTH ) + { + lineLength = 0; + return '\n'; + } // end if + else + { + lineLength++; // This isn't important when decoding + // but throwing an extra "if" seems + // just as wasteful. + + int b = buffer[ position++ ]; + + if( position >= bufferLength ) + position = -1; + + return b & 0xFF; // This is how you "cast" a byte that's + // intended to be unsigned. + } // end else + } // end if: position >= 0 + + // Else error + else + { + // When JDK1.4 is more accepted, use an assertion here. + throw new java.io.IOException( "Error in Base64 code reading stream." ); + } // end else + } // end read + + + /** + * Calls {@link #read()} repeatedly until the end of stream + * is reached or <var>len</var> bytes are read. + * Returns number of bytes read into array or -1 if + * end of stream is encountered. + * + * @param dest array to hold values + * @param off offset for array + * @param len max number of bytes to read into array + * @return bytes read into array or -1 if end of stream is encountered. + * @since 1.3 + */ + public int read( byte[] dest, int off, int len ) throws java.io.IOException + { + int i; + int b; + for( i = 0; i < len; i++ ) + { + b = read(); + + //if( b < 0 && i == 0 ) + // return -1; + + if( b >= 0 ) + dest[off + i] = (byte)b; + else if( i == 0 ) + return -1; + else + break; // Out of 'for' loop + } // end for: each byte read + return i; + } // end read + + } // end inner class InputStream + + + + + + + /* ******** I N N E R C L A S S O U T P U T S T R E A M ******** */ + + + + /** + * A {@link Base64.OutputStream} will write data to another + * <tt>java.io.OutputStream</tt>, given in the constructor, + * and encode/decode to/from Base64 notation on the fly. + * + * @see Base64 + * @since 1.3 + */ + public static class OutputStream extends java.io.FilterOutputStream + { + private boolean encode; + private int position; + private byte[] buffer; + private int bufferLength; + private int lineLength; + private boolean breakLines; + private byte[] b4; // Scratch used in a few places + private boolean suspendEncoding; + + /** + * Constructs a {@link Base64.OutputStream} in ENCODE mode. + * + * @param out the <tt>java.io.OutputStream</tt> to which data will be written. + * @since 1.3 + */ + public OutputStream( java.io.OutputStream out ) + { + this( out, ENCODE ); + } // end constructor + + + /** + * Constructs a {@link Base64.OutputStream} in + * either ENCODE or DECODE mode. + * <p> + * Valid options:<pre> + * ENCODE or DECODE: Encode or Decode as data is read. + * DONT_BREAK_LINES: don't break lines at 76 characters + * (only meaningful when encoding) + * <i>Note: Technically, this makes your encoding non-compliant.</i> + * </pre> + * <p> + * Example: <code>new Base64.OutputStream( out, Base64.ENCODE )</code> + * + * @param out the <tt>java.io.OutputStream</tt> to which data will be written. + * @param options Specified options. + * @see Base64#ENCODE + * @see Base64#DECODE + * @see Base64#DONT_BREAK_LINES + * @since 1.3 + */ + public OutputStream( java.io.OutputStream out, int options ) + { + super( out ); + this.breakLines = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES; + this.encode = (options & ENCODE) == ENCODE; + this.bufferLength = encode ? 3 : 4; + this.buffer = new byte[ bufferLength ]; + this.position = 0; + this.lineLength = 0; + this.suspendEncoding = false; + this.b4 = new byte[4]; + } // end constructor + + + /** + * Writes the byte to the output stream after + * converting to/from Base64 notation. + * When encoding, bytes are buffered three + * at a time before the output stream actually + * gets a write() call. + * When decoding, bytes are buffered four + * at a time. + * + * @param theByte the byte to write + * @since 1.3 + */ + public void write(int theByte) throws java.io.IOException + { + // Encoding suspended? + if( suspendEncoding ) + { + super.out.write( theByte ); + return; + } // end if: supsended + + // Encode? + if( encode ) + { + buffer[ position++ ] = (byte)theByte; + if( position >= bufferLength ) // Enough to encode. + { + out.write( encode3to4( b4, buffer, bufferLength ) ); + + lineLength += 4; + if( breakLines && lineLength >= MAX_LINE_LENGTH ) + { + out.write( NEW_LINE ); + lineLength = 0; + } // end if: end of line + + position = 0; + } // end if: enough to output + } // end if: encoding + + // Else, Decoding + else + { + // Meaningful Base64 character? + if( DECODABET[ theByte & 0x7f ] > WHITE_SPACE_ENC ) + { + buffer[ position++ ] = (byte)theByte; + if( position >= bufferLength ) // Enough to output. + { + int len = Base64.decode4to3( buffer, 0, b4, 0 ); + out.write( b4, 0, len ); + //out.write( Base64.decode4to3( buffer ) ); + position = 0; + } // end if: enough to output + } // end if: meaningful base64 character + else if( DECODABET[ theByte & 0x7f ] != WHITE_SPACE_ENC ) + { + throw new java.io.IOException( "Invalid character in Base64 data." ); + } // end else: not white space either + } // end else: decoding + } // end write + + + + /** + * Calls {@link #write(int)} repeatedly until <var>len</var> + * bytes are written. + * + * @param theBytes array from which to read bytes + * @param off offset for array + * @param len max number of bytes to read into array + * @since 1.3 + */ + public void write( byte[] theBytes, int off, int len ) throws java.io.IOException + { + // Encoding suspended? + if( suspendEncoding ) + { + super.out.write( theBytes, off, len ); + return; + } // end if: supsended + + for( int i = 0; i < len; i++ ) + { + write( theBytes[ off + i ] ); + } // end for: each byte written + + } // end write + + + + /** + * Method added by PHIL. [Thanks, PHIL. -Rob] + * This pads the buffer without closing the stream. + */ + public void flushBase64() throws java.io.IOException + { + if( position > 0 ) + { + if( encode ) + { + out.write( encode3to4( b4, buffer, position ) ); + position = 0; + } // end if: encoding + else + { + throw new java.io.IOException( "Base64 input not properly padded." ); + } // end else: decoding + } // end if: buffer partially full + + } // end flush + + + /** + * Flushes and closes (I think, in the superclass) the stream. + * + * @since 1.3 + */ + public void close() throws java.io.IOException + { + // 1. Ensure that pending characters are written + flushBase64(); + + // 2. Actually close the stream + // Base class both flushes and closes. + super.close(); + + buffer = null; + out = null; + } // end close + + + + /** + * Suspends encoding of the stream. + * May be helpful if you need to embed a piece of + * base640-encoded data in a stream. + * + * @since 1.5.1 + */ + public void suspendEncoding() throws java.io.IOException + { + flushBase64(); + this.suspendEncoding = true; + } // end suspendEncoding + + + /** + * Resumes encoding of the stream. + * May be helpful if you need to embed a piece of + * base640-encoded data in a stream. + * + * @since 1.5.1 + */ + public void resumeEncoding() + { + this.suspendEncoding = false; + } // end resumeEncoding + + + + } // end inner class OutputStream + + +} // end class Base64 Added: amazon-s3/trunk/src/test/java/com/amazon/s3/S3Driver.java =================================================================== --- amazon-s3/trunk/src/test/java/com/amazon/s3/S3Driver.java (rev 0) +++ amazon-s3/trunk/src/test/java/com/amazon/s3/S3Driver.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,139 @@ +package com.amazon.s3; +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + + +public class S3Driver { + + static final String awsAccessKeyId = "1CK9DVFVC6GRDSP6Z9G2"; //<INSERT YOUR AWS ACCESS KEY ID HERE>"; + static final String awsSecretAccessKey = "<INSERT YOUR AWS SECRET ACCESS KEY HERE>"; + + + // convert the bucket to lowercase for vanity domains + // the bucket name must be lowercase since DNS is case-insensitive + static final Bucket bucket = new Bucket(awsAccessKeyId.toLowerCase() + "-test-bucket"); + static final String keyName = "test-key"; + + public static void main(String args[]) throws Exception { + if (awsAccessKeyId.startsWith("<INSERT")) { + System.err.println("Please examine S3Driver.java and update it with your credentials"); + System.exit(-1); + } + + Connection conn = + new Connection(awsAccessKeyId, awsSecretAccessKey); + QueryGenerator generator = + new QueryGenerator(awsAccessKeyId, awsSecretAccessKey); + + // Check if the bucket exists. The high availability engineering of + // Amazon S3 is focused on get, put, list, and delete operations. + // Because bucket operations work against a centralized, global + // resource space, it is not appropriate to make bucket create or + // delete calls on the high availability code path of your application. + // It is better to create or delete buckets in a separate initialization + // or setup routine that you run less often. + if (!conn.exists(bucket)) + { + System.out.println("----- creating bucket -----"); + System.out.println(conn.create(bucket, Connection.LOCATION_DEFAULT, null).getResponseMessage()); + // sample creating an EU located bucket. + // (note path-style urls will not work with location-constrained buckets) + //System.out.println(conn.createBucket(bucketName, AWSAuthConnection.LOCATION_EU, null).connection.getResponseMessage()); + } + + System.out.println("----- listing bucket -----"); + System.out.println(conn.list(bucket).getEntries()); + + System.out.println("----- bucket location -----"); + System.out.println(conn.getLocation(bucket).getLocation()); + + System.out.println("----- putting object -----"); + S3Object object = new S3Object("this is a test".getBytes(), null); + Headers headers = new Headers(); + headers.put("Content-Type", "text/plain"); + System.out.println( + conn.put(bucket, keyName, object, headers).getResponseMessage() + ); + + System.out.println("----- listing bucket -----"); + System.out.println(conn.list(bucket, null, null, null, null).getEntries()); + + System.out.println("----- getting object -----"); + System.out.println( + new String(conn.get(bucket, keyName, null).getObject().getData()) + ); + + System.out.println("----- query string auth example -----"); + generator.setExpiresIn(60 * 1000); + + System.out.println("Try this url in your web browser (it will only work for 60 seconds)\n"); + System.out.println(generator.get(bucket, keyName, null)); + System.out.print("\npress enter> "); + System.in.read(); + + System.out.println("\nNow try just the url without the query string arguments. It should fail.\n"); + System.out.println(generator.makeBareURL(bucket, keyName)); + System.out.print("\npress enter> "); + System.in.read(); + + System.out.println("----- putting object with metadata and public read acl -----"); + + Headers metadata = new Headers(); + metadata.put("blah", "foo"); + object = new S3Object("this is a publicly readable test".getBytes(), new Headers(metadata)); + + headers = new Headers(); + headers.put("x-amz-acl", "public-read"); + headers.put("Content-Type", "text/plain"); + + System.out.println( + conn.put(bucket, keyName + "-public", object, headers).getResponseMessage() + ); + + System.out.println("----- anonymous read test -----"); + System.out.println("\nYou should be able to try this in your browser\n"); + System.out.println(generator.makeBareURL(bucket, keyName + "-public")); + System.out.print("\npress enter> "); + System.in.read(); + + System.out.println("----- path style url example -----"); + System.out.println("\nNon-location-constrained buckets can also be specified as part of the url path. (This was the original url style supported by S3.)"); + System.out.println("\nTry this url out in your browser (it will only be valid for 60 seconds)\n"); + generator.setCallingFormat(CallingFormat.getPathCallingFormat()); + // could also have been done like this: + // generator = new QueryStringAuthGenerator(awsAccessKeyId, awsSecretAccessKey, true, Utils.DEFAULT_HOST, CallingFormat.getPathCallingFormat()); + generator.setExpiresIn(60 * 1000); + System.out.println(generator.get(bucket, keyName, null)); + System.out.print("\npress enter> "); + System.in.read(); + + System.out.println("----- getting object's acl -----"); + System.out.println(new String(conn.getACL(bucket, keyName, null).getObject().getData())); + + System.out.println("----- deleting objects -----"); + System.out.println( + conn.delete(bucket, keyName, null).getResponseMessage() + ); + System.out.println( + conn.delete(bucket, keyName + "-public", null).getResponseMessage() + ); + + System.out.println("----- listing bucket -----"); + System.out.println(conn.list(bucket, null, null, null, null).getEntries()); + + System.out.println("----- listing all my buckets -----"); + System.out.println(conn.listAllBuckets().getEntries()); + + System.out.println("----- deleting bucket -----"); + System.out.println( + conn.delete(bucket).getResponseMessage() + ); + } + +} Added: amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java =================================================================== --- amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java (rev 0) +++ amazon-s3/trunk/src/test/java/com/amazon/s3/S3Test.java 2008-02-06 01:05:04 UTC (rev 5305) @@ -0,0 +1,498 @@ +package com.amazon.s3; +// This software code is made available "AS IS" without warranties of any +// kind. You may copy, display, modify and redistribute the software +// code either by itself or as incorporated into your code; provided that +// you do not remove any proprietary notices. Your use of this software +// code is at your own risk and you waive any claim against Amazon +// Digital Services, Inc. or its affiliates with respect to your use of +// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its +// affiliates. + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class S3Test { + // static final String awsAccessKeyId = "<INSERT YOUR AWS ACCESS KEY ID HERE>"; + // static final String awsSecretAccessKey = "<INSERT YOUR AWS SECRET ACCESS KEY HERE>"; + static final String awsAccessKeyId = "1CK9DVFVC6GRDSP6Z9G2"; //<INSERT YOUR AWS ACCESS KEY ID HERE>"; + static final String awsSecretAccessKey = ""; + + // for subdomains (bucket.s3.amazonaws.com), + // the bucket name must be lowercase since DNS is case-insensitive + static final Bucket bucket = new Bucket(awsAccessKeyId.toLowerCase() + "-test-bucket"); + static int assertionCount = 0; + + static final int UnspecifiedMaxKeys = -1; + + public static void main(String args[]) throws Exception { + if (awsAccessKeyId.startsWith("<INSERT")) { + System.err.println("Please examine S3Test.java and update it with your credentials"); + System.exit(-1); + } + + // test all operation for both regular and vanity domains + // regular: http://s3.amazonaws.com/key + // subdomain: http://bucket.s3.amazonaws.com/key + // testing pure vanity domains (http://<vanity domain>/key) is not covered here + // but is possible with some assitional setup + test(CallingFormat.getSubdomainCallingFormat(), Connection.LOCATION_DEFAULT, true, Connection.DEFAULT_HOST); + test(CallingFormat.getPathCallingFormat(), Connection.LOCATION_DEFAULT, true, Connection.DEFAULT_HOST); + test(CallingFormat.getSubdomainCallingFormat(), Connection.LOCATION_EU, true, Connection.DEFAULT_HOST); + } + + private static void test(CallingFormat format, String location, boolean secure, String server) throws Exception + { + assertionCount = 0; + System.out.println((secure ? "http" : "https") + " / " + server + " / " + + ((location == null) ? "<no-location>" : location) + " / " + + format.getClass().getName()); + + Connection conn = new Connection(awsAccessKeyId, awsSecretAccessKey, secure, server, format); + QueryGenerator generator = + new QueryGenerator(awsAccessKeyId, awsSecretAccessKey, secure, server, format); + + Response response = conn.create(bucket, location, null); + assertEquals( + "couldn't create bucket", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + ListResponse listBucketResponse = conn.list(bucket, null, null, null, null); + assertEquals( + "couldn't get list", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals("list wasn't empty", 0, listBucketResponse.getEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); + + // start delimiter tests + + final String text = "this is a test"; + final String key = "example.txt"; + final String innerKey = "test/inner.txt"; + final String lastKey = "z-last-key.txt"; + + response = conn.put(bucket, key, new S3Object(text.getBytes(), null), null); + assertEquals( + "couldn't put simple object", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + response = conn.put(bucket, innerKey, new S3Object(text.getBytes(), null), null); + assertEquals( + "couldn't put simple object", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + response = conn.put(bucket, lastKey, new S3Object(text.getBytes(), null), null); + assertEquals( + "couldn't put simple object", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + // plain list + listBucketResponse = conn.list(bucket, null, null, null, null); + assertEquals( + "couldn't get list", + listBucketResponse.getResponseCode(), + HttpURLConnection.HTTP_OK); + assertEquals("Unexpected list size", 3, listBucketResponse.getEntries().size()); + assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); + + // root "directory" + listBucketResponse = conn.list(bucket, null, null, null, "/", null); + assertEquals( + "couldn't get list", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals("Unexpected list size", 2, listBucketResponse.getEntries().size()); + assertEquals("Unexpected common prefix size", 1, listBucketResponse.getCommonPrefixEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, "/", false, null); + + // root "directory" with a max-keys of "1" + listBucketResponse = conn.list(bucket, null, null, new Integer( 1 ), "/", null); + assertEquals( + "couldn't get list", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); + assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 1, "/", true, "example.txt"); + + // root "directory" with a max-keys of "2" + listBucketResponse = conn.list(bucket, null, null, new Integer( 2 ), "/", null); + assertEquals( + "couldn't get list", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); + assertEquals("Unexpected common prefix size", 1, listBucketResponse.getCommonPrefixEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 2, "/", true, "test/"); + String marker = listBucketResponse.getNextMarker(); + listBucketResponse = conn.list(bucket, null, marker, new Integer( 2 ), "/", null); + assertEquals( + "couldn't get list", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); + assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", marker, 2, "/", false, null); + + // test "directory" + listBucketResponse = conn.list(bucket, "test/", null, null, "/", null); + assertEquals( + "couldn't get list", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); + assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "test/", "", UnspecifiedMaxKeys, "/", false, null); + + // remove innerkey + response = conn.delete(bucket, innerKey, null); + assertEquals( + "couldn't delete entry", + HttpURLConnection.HTTP_NO_CONTENT, + response.getResponseCode()); + + // remove last key + response = conn.delete(bucket, lastKey, null); + assertEquals( + "couldn't delete entry", + HttpURLConnection.HTTP_NO_CONTENT, + response.getResponseCode()); + + + // end delimiter tests + + response = conn.put(bucket, key, new S3Object(text.getBytes(), null), null); + assertEquals( + "couldn't reput simple object", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + Map metadata = new HashMap(); + metadata.put("title", Arrays.asList(new String[] { "title" })); + response = conn.put(bucket, key, new S3Object(text.getBytes(), new Headers(metadata)), null); + assertEquals( + "couldn't put complex object", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + GetResponse getResponse = conn.get(bucket, key, null); + assertEquals( + "couldn't get object", + getResponse.getResponseCode(), + HttpURLConnection.HTTP_OK); + assertEquals("didn't get the right data back", text.getBytes(), getResponse.getObject().getData()); + assertEquals("didn't get the right metadata back", 1, getResponse.getObject().getMetadata().size()); + assertEquals( + "didn't get the right metadata back", + 1, + ((List)getResponse.getObject().getMetadata().getValues("title")).size()); + assertEquals( + "didn't get the right metadata back", + "title", + ((List)getResponse.getObject().getMetadata().getValues("title")).get(0)); + assertEquals( + "didn't get the right content-length", + ""+text.length(), + getResponse.getHeaderField("Content-Length")); + + + String titleWithSpaces = " \t title with leading and trailing spaces "; + Headers h = new Headers(); + h.put("title", titleWithSpaces); + response = conn.put(bucket, key, new S3Object(text.getBytes(), h), null); + assertEquals( + "couldn't put metadata with leading and trailing spaces", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + getResponse = conn.get(bucket, key, null); + assertEquals( + "couldn't get object", + HttpURLConnection.HTTP_OK, + getResponse.getResponseCode()); + assertEquals("didn't get the right metadata back", getResponse.getObject().getMetadata().size(), 1); + assertEquals( + "didn't get the right metadata back", + 1, + ((List)getResponse.getObject().getMetadata().getValues("title")).size()); + assertEquals( + "didn't get the right metadata back", + titleWithSpaces.trim(), + ((List)getResponse.getObject().getMetadata().getValues("title")).get(0)); + + String weirdKey = "&=//%# ++++"; + response = conn.put(bucket, weirdKey, new S3Object(text.getBytes(), null), null); + assertEquals( + "couldn't put weird key", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + getResponse = conn.get(bucket, weirdKey, null); + assertEquals( + "couldn't get weird key", + HttpURLConnection.HTTP_OK, + getResponse.getResponseCode()); + + // start acl test + + getResponse = conn.getACL(bucket, key, null); + assertEquals( + "couldn't get acl", + HttpURLConnection.HTTP_OK, + getResponse.getResponseCode()); + + byte[] acl = getResponse.getObject().getData(); + + response = conn.putACL(bucket, key, new String(acl), null); + assertEquals( + "couldn't put acl", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + getResponse = conn.getACL(bucket, null); + assertEquals( + "couldn't get bucket acl", + HttpURLConnection.HTTP_OK, + getResponse.getResponseCode()); + + byte[] bucketACL = getResponse.getObject().getData(); + + response = conn.putACL(bucket, new String(bucketACL), null); + assertEquals( + "couldn't put bucket acl", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + // end acl test + + // bucket logging tests + getResponse = conn.getBucketLogging(bucket, null); + assertEquals( + "couldn't get bucket logging config", + HttpURLConnection.HTTP_OK, + getResponse.getResponseCode()); + + byte[] bucketLogging = getResponse.getObject().getData(); + + response = conn.putBucketLogging(bucket, new String(bucketLogging), null); + assertEquals( + "couldn't put bucket logging config", + HttpURLConnection.HTTP_OK, + response.getResponseCode()); + + // end bucket logging tests + + listBucketResponse = conn.list(bucket, null, null, null, null); + assertEquals( + "couldn't list bucket", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + List entries = listBucketResponse.getEntries(); + assertEquals("didn't get back the right number of entries", 2, entries.size()); + // depends on weirdKey < $key + assertEquals("first key isn't right", weirdKey, ((Entry)entries.get(0)).getKey()); + assertEquals("second key isn't right", key, ((Entry)entries.get(1)).getKey()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); + + listBucketResponse = conn.list(bucket, null, null, new Integer(1), null); + assertEquals( + "couldn't list bucket", + HttpURLConnection.HTTP_OK, + listBucketResponse.getResponseCode()); + assertEquals( + "didn't get back the right number of entries", + 1, + listBucketResponse.getEntries().size()); + verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 1, null, true, null); + + for (Iterator it = entries.iterator(); it.hasNext(); ) { + Entry entry = (Entry)it.next(); + response = conn.delete(bucket, entry.getKey(), null); + assertEquals( + "couldn't delete entry", + HttpURLConnection.HTTP_NO_CONTENT, + response.getResponseCode()); + } + + ListAllBucketsResponse listAllMyBucketsResponse = conn.listAllBuckets(); + assertEquals( + "couldn't list all my buckets", + HttpURLConnection.HTTP_OK, + listAllMyBucketsResponse.getResponseCode()); + List buckets = listAllMyBucketsResponse.getEntries(); + + response = conn.delete(bucket); + assertEquals( + "couldn't delete bucket", + HttpURLConnection.HTTP_NO_CONTENT, + response.getResponseCode()); + + listAllMyBucketsResponse = conn.listAllBuckets(); + assertEquals( + "couldn't list all my buckets", + HttpURLConnection.HTTP_OK, + listAllMyBucketsResponse.getResponseCode()); + assertEquals( + "bucket count is incorrect", + buckets.size() - 1, + listAllMyBucketsResponse.getEntries().size()); + + checkURL( + generator.create(bucket, null), + "PUT", + HttpURLConnection.HTTP_OK, + "couldn't create bucket"); + checkURL( + generator.put(bucket, key, new S3Object("test data".getBytes(), null), null), + "PUT", + HttpURLConnection.HTTP_OK, + "put object", + "test data"); + checkURL( + generator.get(bucket, key, null), + "GET", + HttpURLConnection.HTTP_OK, + "get object"); + checkURL( + generator.list(bucket, null, null, null, null), + "GET", + HttpURLConnection.HTTP_OK, + "list bucket"); + checkURL( + generator.listAllBuckets(), + "GET", + HttpURLConnection.HTTP_OK, + "list all my buckets"); + checkURL( + generator.getACL(bucket, key, null), + "GET", + HttpURLConnection.HTTP_OK, + "get acl"); + checkURL( + generator.putACL(bucket, key, new String(acl), null), + "PUT", + HttpURLConnection.HTTP_OK, + "put acl", + new String(acl)); + checkURL( + generator.getACL(bucket, null), + "GET", + HttpURLConnection.HTTP_OK, + "get bucket acl"); + checkURL( + generator.putACL(bucket, new String(bucketACL), null), + "PUT", + HttpURLConnection.HTTP_OK, + "put bucket acl", + new String(bucketACL)); + checkURL( + generator.getBucketLogging(bucket, null), + "GET", + HttpURLConnection.HTTP_OK, + "get bucket logging"); + checkURL( + generator.putBucketLogging(bucket, new String(bucketLogging), null), + "PUT", + HttpURLConnection.HTTP_OK, + "put bucket logging", + new String(bucketLogging)); + checkURL( + generator.delete(bucket, key, null), + "DELETE", + HttpURLConnection.HTTP_NO_CONTENT, + "delete object"); + checkURL( + generator.delete(bucket, null), + "DELETE", + HttpURLConnection.HTTP_NO_CONTENT, + "delete bucket"); + + System.out.println("OK (" + assertionCount + " tests passed)"); + } + + private static void verifyBucketResponseParameters( ListResponse listBucketResponse, + Bucket bucket, String prefix, String marker, + int maxKeys, String delimiter, boolean isTruncated, + String nextMarker ) { + assertEquals("Bucket name should match.", bucket.getName(), listBucketResponse.getName()); + assertEquals("Bucket prefix should match.", prefix, listBucketResponse.getPrefix()); + assertEquals("Bucket marker should match.", marker, listBucketResponse.getMarker()); + assertEquals("Bucket delimiter should match.", delimiter, listBucketResponse.getDelimiter()); + if ( UnspecifiedMaxKeys != maxKeys ) { + assertEquals("Bucket max-keys should match.", maxKeys, listBucketResponse.getMaxKeys()); + } + assertEquals("Bucket should not be truncated.", isTruncated, listBucketResponse.isTruncated()); + assertEquals("Bucket nextMarker should match.", nextMarker, listBucketResponse.getNextMarker()); + } + + + private static void assertEquals(String message, int expected, int actual) { + assertionCount++; + if (expected != actual) { + throw new RuntimeException(message + ": expected " + expected + " but got " + actual); + } + } + + private static void assertEquals(String message, byte[] expected, byte[] actual) { + assertionCount++; + if (! Arrays.equals(expected, actual)) { + throw new RuntimeException( + message + + ": expected " + + new String(expected) + + " but got " + + new String(actual)); + } + } + + private static void assertEquals(String message, Object expected, Object actual) { + assertionCount++; + if (expected != actual && (actual == null || ! actual.equals(expected))) { + throw new RuntimeException(message + ": expected " + expected + " but got " + actual); + } + } + + private static void assertEquals(String message, boolean expected, boolean actual) { + assertionCount++; + if (expected != actual) { + throw new RuntimeException(message + ": expected " + expected + " but got " + actual); + } + } + + private static void checkURL(URL url, String method, int code, String message) + throws MalformedURLException, IOException + { + checkURL(url, method, code, message, null); + } + + private static void checkURL(URL url, String method, int code, String message, String data) + throws MalformedURLException, IOException + { + if (data == null) data = ""; + + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setRequestMethod(method); + if ("PUT".equals(method)) { + connection.setRequestProperty("Content-Length", ""+data.getBytes().length); + connection.setDoOutput(true); + connection.getOutputStream().write(data.getBytes()); + } else { + // HttpURLConnection auto populates Content-Type, which we don't want here + connection.setRequestProperty("Content-Type", ""); + } + + assertEquals(message, code, connection.getResponseCode()); + } +}