support specifying prefix and dest-prefix inline with bucket names

README.md

@@ -40,7 +40,7 @@ The above command requires that Maven 3 is installed.
 
 ### Usage
 
-    s3s3mirror.sh [options] <source-bucket> <destination-bucket>
+    s3s3mirror.sh [options] <source-bucket>[/src-prefix/path/...] <destination-bucket>[/dest-prefix/path/...]
 
 ### Options
 
@@ -62,26 +62,33 @@ Copy everything from a bucket named "source" to another bucket named "dest"
 
     s3s3mirror.sh source dest
 
-Copy everything from "source" to "dest", but only copy objects created within the past week (these are equivalent)
+Copy everything from "source" to "dest", but only copy objects created within the past week
 
     s3s3mirror.sh -c 7 source dest
     s3s3mirror.sh -c 7d source dest
     s3s3mirror.sh -c 1w source dest
+    s3s3mirror.sh --ctime 1w source dest
 
 Copy everything from "source/foo" to "dest/bar"
 
+    s3s3mirror.sh source/foo dest/bar
     s3s3mirror.sh -p foo -d bar source dest
 
 Copy everything from "source/foo" to "dest/bar" and delete anything in "dest/bar" that does not exist in "source/foo"
 
+    s3s3mirror.sh -X source/foo dest/bar
+    s3s3mirror.sh --delete-removed source/foo dest/bar
     s3s3mirror.sh -p foo -d bar -X source dest
+    s3s3mirror.sh -p foo -d bar --delete-removed source dest
 
 Copy within a single bucket -- copy everything from "source/foo" to "source/bar"
 
+    s3s3mirror.sh source/foo source/bar
     s3s3mirror.sh -p foo -d bar source source
 
 BAD IDEA: If copying within a single bucket, do *not* put the destination below the source
 
+    s3s3mirror.sh source/foo source/foo/subfolder
     s3s3mirror.sh -p foo -d foo/subfolder source source
-*This is likely to cause infinite recursion and send your AWS bill into the stratosphere!*
+*This might cause recursion and raise your AWS bill unnecessarily*
 

pom.xml

@@ -11,7 +11,7 @@
 
     <groupId>org.cobbzilla</groupId>
     <artifactId>s3s3mirror</artifactId>
-    <version>1.2.1-SNAPSHOT</version>
+    <version>1.2.2-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <licenses>

s3s3mirror.bat

@@ -1,2 +1,2 @@
 @echo off
-java -Ds3s3mirror.version=1.2.1 -jar target/s3s3mirror-1.2.1-SNAPSHOT.jar %*
+java -Ds3s3mirror.version=1.2.2 -jar target/s3s3mirror-1.2.2-SNAPSHOT.jar %*

s3s3mirror.sh

@@ -4,7 +4,7 @@ THISDIR=$(dirname $0)
 cd ${THISDIR}
 THISDIR=$(pwd)
 
-VERSION=1.2.1
+VERSION=1.2.2
 JARFILE=target/s3s3mirror-${VERSION}-SNAPSHOT.jar
 VERSION_ARG="-Ds3s3mirror.version=${VERSION}"
 

src/main/java/org/cobbzilla/s3s3mirror/KeyCopyJob.java

@@ -53,8 +53,8 @@ public void run() {
                 for (int tries=0; tries<maxRetries; tries++) {
                     if (verbose) log.info("copying (try #"+tries+"): "+key+" to: "+keydest);
                     try {
-                        client.copyObject(request);
                         stats.s3copyCount.incrementAndGet();
+                        client.copyObject(request);
                         stats.bytesCopied.addAndGet(sourceMetadata.getContentLength());
                         copiedOK = true;
                         if (verbose) log.info("successfully copied (on try #"+tries+"): "+key+" to: "+keydest);

src/main/java/org/cobbzilla/s3s3mirror/KeyDeleteJob.java

@@ -42,8 +42,8 @@ public void run() {
                 for (int tries=0; tries<maxRetries; tries++) {
                     if (verbose) log.info("deleting (try #"+tries+"): "+key);
                     try {
-                        client.deleteObject(request);
                         stats.s3deleteCount.incrementAndGet();
+                        client.deleteObject(request);
                         deletedOK = true;
                         if (verbose) log.info("successfully deleted (on try #"+tries+"): "+key);
                         break;

src/main/java/org/cobbzilla/s3s3mirror/KeyJob.java

@@ -29,9 +29,8 @@ protected ObjectMetadata getObjectMetadata(String bucket, String key, MirrorOpti
         Exception ex = null;
         for (int tries=0; tries<options.getMaxRetries(); tries++) {
             try {
-                final ObjectMetadata objectMetadata = client.getObjectMetadata(bucket, key);
                 context.getStats().s3getCount.incrementAndGet();
-                return objectMetadata;
+                return client.getObjectMetadata(bucket, key);
 
             } catch (AmazonS3Exception e) {
                 if (e.getStatusCode() == 404) throw e;
@@ -55,9 +54,8 @@ protected AccessControlList getAccessControlList(MirrorOptions options, String k
         Exception ex = null;
         for (int tries=0; tries<options.getMaxRetries(); tries++) {
             try {
-                final AccessControlList objectAcl = client.getObjectAcl(options.getSourceBucket(), key);
                 context.getStats().s3getCount.incrementAndGet();
-                return objectAcl;
+                return client.getObjectAcl(options.getSourceBucket(), key);
 
             } catch (Exception e) {
                 ex = e;

src/main/java/org/cobbzilla/s3s3mirror/KeyLister.java

@@ -34,8 +34,7 @@ public KeyLister(AmazonS3Client client, MirrorContext context, int maxQueueCapac
         this.summaries = new ArrayList<S3ObjectSummary>(10*fetchSize);
 
         final ListObjectsRequest request = new ListObjectsRequest(bucket, prefix, null, null, fetchSize);
-        listing = client.listObjects(request);
-        context.getStats().s3getCount.incrementAndGet();
+        listing = s3getFirstBatch(client, request);
         synchronized (summaries) {
             final List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
             summaries.addAll(objectSummaries);
@@ -84,15 +83,38 @@ public void run() {
         }
     }
 
+    private ObjectListing s3getFirstBatch(AmazonS3Client client, ListObjectsRequest request) {
+
+        final MirrorOptions options = context.getOptions();
+        final boolean verbose = options.isVerbose();
+        final int maxRetries = options.getMaxRetries();
+
+        Exception lastException = null;
+        for (int tries=0; tries<maxRetries; tries++) {
+            try {
+                context.getStats().s3getCount.incrementAndGet();
+                ObjectListing listing = client.listObjects(request);
+                if (verbose) log.info("successfully got first batch of objects (on try #"+tries+")");
+                return listing;
+
+            } catch (Exception e) {
+                lastException = e;
+                log.warn("s3getFirstBatch: error listing (try #"+tries+"): "+e);
+                Sleep.sleep(50);
+            }
+        }
+        throw new IllegalStateException("s3getFirstBatch: error listing: "+lastException, lastException);
+    }
+
     private ObjectListing s3getNextBatch() {
         final MirrorOptions options = context.getOptions();
         final boolean verbose = options.isVerbose();
         final int maxRetries = options.getMaxRetries();
 
         for (int tries=0; tries<maxRetries; tries++) {
             try {
-                ObjectListing next = client.listNextBatchOfObjects(listing);
                 context.getStats().s3getCount.incrementAndGet();
+                ObjectListing next = client.listNextBatchOfObjects(listing);
                 if (verbose) log.info("successfully got next batch of objects (on try #"+tries+")");
                 return next;
 

src/main/java/org/cobbzilla/s3s3mirror/KeyMaster.java

@@ -75,14 +75,14 @@ public void run() {
 
         final int maxQueueCapacity = MirrorMaster.getMaxQueueCapacity(options);
 
-        final KeyLister lister = new KeyLister(client, context, maxQueueCapacity, getBucket(options), getPrefix(options));
-        executorService.submit(lister);
-
-        List<S3ObjectSummary> summaries = lister.getNextBatch();
-        if (verbose) log.info(summaries.size()+" keys found in first batch from source bucket -- processing...");
-
         int counter = 0;
         try {
+            final KeyLister lister = new KeyLister(client, context, maxQueueCapacity, getBucket(options), getPrefix(options));
+            executorService.submit(lister);
+
+            List<S3ObjectSummary> summaries = lister.getNextBatch();
+            if (verbose) log.info(summaries.size()+" keys found in first batch from source bucket -- processing...");
+
             while (true) {
                 for (S3ObjectSummary summary : summaries) {
                     while (workQueue.size() >= maxQueueCapacity) {

src/main/java/org/cobbzilla/s3s3mirror/MirrorMaster.java

@@ -27,7 +27,7 @@ public void mirror() {
 
         final MirrorOptions options = context.getOptions();
 
-        if (options.isVerbose()) log.info("will not copy anything older than "+options.getCtime()+" (cutoff="+options.getMaxAgeDate()+")");
+        if (options.isVerbose() && options.hasCtime()) log.info("will not copy anything older than "+options.getCtime()+" (cutoff="+options.getMaxAgeDate()+")");
 
         final int maxQueueCapacity = getMaxQueueCapacity(options);
         final BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>(maxQueueCapacity);

src/main/java/org/cobbzilla/s3s3mirror/MirrorOptions.java

@@ -110,18 +110,42 @@ private int getCtimeNumber(String ctime) {
     @Option(name=OPT_DELETE_REMOVED, aliases=LONGOPT_DELETE_REMOVED, usage=USAGE_DELETE_REMOVED)
     @Getter @Setter private boolean deleteRemoved = false;
 
+    @Argument(index=0, required=true, usage="source bucket[/source/prefix]") @Getter @Setter private String source;
+    @Argument(index=1, required=true, usage="destination bucket[/dest/prefix]") @Getter @Setter private String destination;
+
+    @Getter private String sourceBucket;
+    @Getter private String destinationBucket;
+
     public void initDerivedFields() {
+
         if (hasCtime()) {
             this.maxAge = initMaxAge();
             this.maxAgeDate = new Date(maxAge).toString();
         }
-    }
 
-    @Argument(index=0, required=true, usage="source bucket") @Getter @Setter private String source;
-    @Argument(index=1, required=true, usage="destination bucket") @Getter @Setter private String destination;
+        String scrubbed;
+        int slashPos;
+
+        scrubbed = scrubS3ProtocolPrefix(source);
+        slashPos = scrubbed.indexOf('/');
+        if (slashPos == -1) {
+            sourceBucket = scrubbed;
+        } else {
+            sourceBucket = scrubbed.substring(0, slashPos);
+            if (hasPrefix()) throw new IllegalArgumentException("Cannot use a "+OPT_PREFIX+"/"+LONGOPT_PREFIX+" argument and source path that includes a prefix at the same time");
+            prefix = scrubbed.substring(slashPos+1);
+        }
 
-    public String getSourceBucket() { return scrubS3ProtocolPrefix(getSource()); }
-    public String getDestinationBucket() { return scrubS3ProtocolPrefix(getDestination()); }
+        scrubbed = scrubS3ProtocolPrefix(destination);
+        slashPos = scrubbed.indexOf('/');
+        if (slashPos == -1) {
+            destinationBucket = scrubbed;
+        } else {
+            destinationBucket = scrubbed.substring(0, slashPos);
+            if (hasDestPrefix()) throw new IllegalArgumentException("Cannot use a "+OPT_DEST_PREFIX+"/"+LONGOPT_DEST_PREFIX+" argument and destination path that includes a dest-prefix at the same time");
+            destPrefix = scrubbed.substring(slashPos+1);
+        }
+    }
 
     protected String scrubS3ProtocolPrefix(String bucket) {
         bucket = bucket.trim();
@@ -130,5 +154,4 @@ protected String scrubS3ProtocolPrefix(String bucket) {
         }
         return bucket;
     }
-
 }

src/test/java/org/cobbzilla/s3s3mirror/MirrorMainTest.java

@@ -46,4 +46,62 @@ public void testMaxConnectionsArgs () throws Exception {
         assertEquals(SOURCE, options.getSource());
         assertEquals(DESTINATION, options.getDestination());
     }
+
+    @Test
+    public void testInlinePrefix () throws Exception {
+        final String prefix = "foo";
+        final MirrorMain main = new MirrorMain(new String[]{SOURCE+"/"+prefix, DESTINATION});
+        main.parseArguments();
+
+        final MirrorOptions options = main.getOptions();
+        assertEquals(prefix, options.getPrefix());
+        assertNull(options.getDestPrefix());
+    }
+
+    @Test
+    public void testInlineDestPrefix () throws Exception {
+        final String destPrefix = "foo";
+        final MirrorMain main = new MirrorMain(new String[]{SOURCE, DESTINATION+"/"+destPrefix});
+        main.parseArguments();
+
+        final MirrorOptions options = main.getOptions();
+        assertEquals(destPrefix, options.getDestPrefix());
+        assertNull(options.getPrefix());
+    }
+
+    @Test
+    public void testInlineSourceAndDestPrefix () throws Exception {
+        final String prefix = "foo";
+        final String destPrefix = "bar";
+        final MirrorMain main = new MirrorMain(new String[]{SOURCE+"/"+prefix, DESTINATION+"/"+destPrefix});
+        main.parseArguments();
+
+        final MirrorOptions options = main.getOptions();
+        assertEquals(prefix, options.getPrefix());
+        assertEquals(destPrefix, options.getDestPrefix());
+    }
+
+    @Test
+    public void testInlineSourcePrefixAndPrefixOption () throws Exception {
+        final String prefix = "foo";
+        final MirrorMain main = new MirrorMain(new String[]{MirrorOptions.OPT_PREFIX, prefix, SOURCE+"/"+prefix, DESTINATION});
+        try {
+            main.parseArguments();
+            fail("expected IllegalArgumentException");
+        } catch (Exception e) {
+            assertTrue(e instanceof IllegalArgumentException);
+        }
+    }
+
+    @Test
+    public void testInlineDestinationPrefixAndPrefixOption () throws Exception {
+        final String prefix = "foo";
+        final MirrorMain main = new MirrorMain(new String[]{MirrorOptions.OPT_DEST_PREFIX, prefix, SOURCE, DESTINATION+"/"+prefix});
+        try {
+            main.parseArguments();
+            fail("expected IllegalArgumentException");
+        } catch (Exception e) {
+            assertTrue(e instanceof IllegalArgumentException);
+        }
+    }
 }