Add support for fetching directories

get_s3.go

@@ -17,7 +17,75 @@ import (
 type S3Getter struct{}
 
 func (g *S3Getter) Get(dst string, u *url.URL) error {
-	return fmt.Errorf("Operation is unsupported")
+	// Parse URL
+	region, bucket, path, err := g.parseUrl(u)
+	if err != nil {
+		return err
+	}
+
+	// Remove destination if it already exists
+	_, err = os.Stat(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if err == nil {
+		// Remove the destination
+		if err := os.RemoveAll(dst); err != nil {
+			return err
+		}
+	}
+
+	// Create all the parent directories
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	client := s3.New(&aws.Config{Region: aws.String(region)})
+
+	// List files in path, keep listing until no more objects are found
+	lastMarker := ""
+	hasMore := true
+	for hasMore {
+		req := &s3.ListObjectsInput{
+			Bucket: aws.String(bucket),
+			Prefix: aws.String(path),
+		}
+
+		if lastMarker != "" {
+			req.Marker = aws.String(lastMarker)
+		}
+
+		resp, err := client.ListObjects(req)
+		if err != nil {
+			return err
+		}
+
+		hasMore = aws.BoolValue(resp.IsTruncated)
+
+		for _, object := range resp.Contents {
+			lastMarker = aws.StringValue(object.Key)
+			objPath := aws.StringValue(object.Key)
+
+			// If the key ends with a backslash assume it is a directory and ignore
+			if strings.HasSuffix(objPath, "/") {
+				continue
+			}
+
+			// Get the object destination path
+			objDst, err := filepath.Rel(path, objPath)
+			if err != nil {
+				return err
+			}
+			objDst = filepath.Join(dst, objDst)
+
+			if err := g.getObject(client, objDst, bucket, objPath); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
 }
 
 func (g *S3Getter) GetFile(dst string, u *url.URL) error {
@@ -27,9 +95,14 @@ func (g *S3Getter) GetFile(dst string, u *url.URL) error {
 	}
 
 	client := s3.New(&aws.Config{Region: aws.String(region)})
+
+	return g.getObject(client, dst, bucket, path)
+}
+
+func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key string) error {
 	resp, err := client.GetObject(&s3.GetObjectInput{
 		Bucket: aws.String(bucket),
-		Key:    aws.String(path),
+		Key:    aws.String(key),
 	})
 	if err != nil {
 		return err

get_s3_test.go

@@ -0,0 +1,59 @@
+package getter
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestS3Getter_impl(t *testing.T) {
+	var _ Getter = new(S3Getter)
+}
+
+func TestS3Getter(t *testing.T) {
+	g := new(S3Getter)
+	dst := tempDir(t)
+
+	// With a dir that doesn't exist
+	if err := g.Get(dst, testURL("https://s3-eu-west-1.amazonaws.com/hailo-s3-test")); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	mainPath := filepath.Join(dst, "main.tf")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestS3Getter_Subdir(t *testing.T) {
+	g := new(S3Getter)
+	dst := tempDir(t)
+
+	// With a dir that doesn't exist
+	if err := g.Get(dst, testURL("https://s3-eu-west-1.amazonaws.com/hailo-s3-test/subdir")); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	subPath := filepath.Join(dst, "sub.tf")
+	if _, err := os.Stat(subPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestS3Getter_GetFile(t *testing.T) {
+	g := new(S3Getter)
+	dst := tempFile(t)
+
+	// Download
+	if err := g.GetFile(dst, testURL("https://s3-eu-west-1.amazonaws.com/hailo-s3-test/foo.txt")); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	if _, err := os.Stat(dst); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	assertContents(t, dst, "Hello\n")
+}

module_test.go

@@ -58,6 +58,15 @@ func testModuleURL(n string) *url.URL {
 	return u
 }
 
+func testURL(s string) *url.URL {
+	u, err := urlhelper.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return u
+}
+
 func testStorage(t *testing.T) Storage {
 	return &FolderStorage{StorageDir: tempDir(t)}
 }