added new cache error types and implemented them in the s3cache. adde…

…d support for s3cach basepath. updated s3cache tests to use env vars. go-spatial#211

cache/errors.go

@@ -20,3 +20,30 @@ type ErrInvalidFileKey struct {
 func (e ErrInvalidFileKey) Error() string {
 	return fmt.Sprintf("cache: invalid fileKey (%v). unable to parse (%v) value (%v) into int", e.path, e.key, e.val)
 }
+
+type ErrGettingFromCache struct {
+	Err       error
+	CacheType string
+}
+
+func (e ErrGettingFromCache) Error() string {
+	return fmt.Sprintf("cache: error getting from (%v) cache: %v", e.CacheType, e.Err)
+}
+
+type ErrSettingToCache struct {
+	Err       error
+	CacheType string
+}
+
+func (e ErrSettingToCache) Error() string {
+	return fmt.Sprintf("cache: error setting to (%v) cache: %v", e.CacheType, e.Err)
+}
+
+type ErrPurgingCache struct {
+	Err       error
+	CacheType string
+}
+
+func (e ErrPurgingCache) Error() string {
+	return fmt.Sprintf("cache: error purging (%v) cache: %v", e.CacheType, e.Err)
+}

cache/s3cache/README.md

@@ -1,6 +1,6 @@
 # S3Cache
 
-s3cache is an abstration on top of Amazon Web Services (AWS) Simple Storage Service (S3) which implements the tegola cache interface. To use it, add the following minimum config to your tegola config file:
+s3cache is an abstraction on top of Amazon Web Services (AWS) Simple Storage Service (S3) which implements the tegola cache interface. To use it, add the following minimum config to your tegola config file:
 
 ```toml
 [cache]
@@ -9,12 +9,13 @@ bucket="tegola-test-data"
 ```
 
 ## Properties
-The s3cache config supports the following config properties:
+The s3cache config supports the following properties:
 
 - `bucket` (string): [Required] the name of the S3 bucket to use.
-- `region` (string): [Optoinal] the region the bucket is in. Defaults to 'us-east-1'
-- `aws_access_key_id` (string): [Optoinal] the AWS access key id to use.
-- `aws_secret_access_key` (string): [Optoinal] the AWS secret access key to use.
+- `basepath` (string): [Optional] a path prefix added to all cache operations inside of the S3 bucket. helpful so a bucket does not need to be dedicated to only this cache.
+- `region` (string): [Optional] the region the bucket is in. Defaults to 'us-east-1'
+- `aws_access_key_id` (string): [Optional] the AWS access key id to use.
+- `aws_secret_access_key` (string): [Optional] the AWS secret access key to use.
 - `max_zoom` (int): [Optional] the max zoom the cache should cache to. After this zoom, Set() calls will return before doing work.
 
 ## Credential chain
@@ -25,3 +26,14 @@ $ export AWS_REGION=us-west-2
 $ export AWS_ACCESS_KEY_ID=YOUR_AKID
 $ export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY
 ```
+
+## Testing
+Testing is designed to work against a live S3 bucket. To run the s3 cache tests, the following environment variables need to be set:
+
+```bash
+$ export RUN_S3_TESTS=yes
+$ export AWS_TEST_BUCKET=YOUR_TEST_BUCKET_NAME
+$ export AWS_REGION=TEST_BUCKET_REGION
+$ export AWS_ACCESS_KEY_ID=YOUR_AKID
+$ export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY
+```

cache/s3cache/s3cache.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -26,6 +27,7 @@ const (
 	//	required
 	ConfigKeyBucket = "bucket"
 	//	optional
+	ConfigKeyBasepath       = "basepath"
 	ConfigKeyMaxZoom        = "max_zoom"
 	ConfigKeyRegion         = "region" //	defaults to "us-east-1"
 	ConfigKeyAWSAccessKeyID = "aws_access_key_id"
@@ -49,6 +51,7 @@ func init() {
 //			region (string): the AWS region the bucket is located. defaults to 'us-east-1'
 //			aws_access_key_id (string): an AWS access key id
 //			aws_secret_access_key (string): an AWS secret access key
+//			basepath (string): a path prefix added to all cache operations inside of the S3 bucket
 //			max_zoom (int): max zoom to use the cache. beyond this zoom cache Set() calls will be ignored
 
 func New(config map[string]interface{}) (cache.Interface, error) {
@@ -78,6 +81,13 @@ func New(config map[string]interface{}) (cache.Interface, error) {
 		return nil, ErrMissingBucket
 	}
 
+	//	basepath
+	basepath := ""
+	s3cache.Basepath, err = c.String(ConfigKeyBasepath, &basepath)
+	if err != nil {
+		return nil, err
+	}
+
 	//	check for region env var
 	region := os.Getenv("AWS_REGION")
 	if region == "" {
@@ -127,21 +137,36 @@ func New(config map[string]interface{}) (cache.Interface, error) {
 	}
 	//	write a test file
 	if err := s3cache.Set(&key, []byte("\x53\x69\x6c\x61\x73")); err != nil {
-		return nil, err
+		e := cache.ErrSettingToCache{
+			CacheType: CacheType,
+			Err:       err,
+		}
+
+		return nil, e
 	}
 
 	//	read the test file
 	_, hit, err := s3cache.Get(&key)
 	if err != nil {
-		return nil, err
+		e := cache.ErrGettingFromCache{
+			CacheType: CacheType,
+			Err:       err,
+		}
+
+		return nil, e
 	}
 	if !hit {
 		//	return an error?
 	}
 
 	//	purge the test file
 	if err := s3cache.Purge(&key); err != nil {
-		return nil, err
+		e := cache.ErrPurgingCache{
+			CacheType: CacheType,
+			Err:       err,
+		}
+
+		return nil, e
 	}
 
 	return &s3cache, nil
@@ -151,6 +176,10 @@ type S3Cache struct {
 	//	Bucket is the name of the s3 bucket to operate on
 	Bucket string
 
+	//	Basepath is a path prefix added to all cache operations inside of the S3 bucket
+	//	helpful so a bucket does not need to be dedicated to only this cache
+	Basepath string
+
 	//	MaxZoom determins the max zoom the cache to persist. Beyond this
 	//	zoom, cache Set() calls will be ignored. This is useful if the cache
 	//	should not be leveraged for higher zooms when data changes often.
@@ -169,10 +198,13 @@ func (s3c *S3Cache) Set(key *cache.Key, val []byte) error {
 		return nil
 	}
 
+	//	add our basepath
+	k := filepath.Join(s3c.Basepath, key.String())
+
 	input := s3.PutObjectInput{
 		Body:   aws.ReadSeekCloser(bytes.NewReader(val)),
 		Bucket: aws.String(s3c.Bucket),
-		Key:    aws.String(key.String()),
+		Key:    aws.String(k),
 	}
 
 	_, err = s3c.Client.PutObject(&input)
@@ -186,9 +218,12 @@ func (s3c *S3Cache) Set(key *cache.Key, val []byte) error {
 func (s3c *S3Cache) Get(key *cache.Key) ([]byte, bool, error) {
 	var err error
 
+	//	add our basepath
+	k := filepath.Join(s3c.Basepath, key.String())
+
 	input := s3.GetObjectInput{
 		Bucket: aws.String(s3c.Bucket),
-		Key:    aws.String(key.String()),
+		Key:    aws.String(k),
 	}
 
 	result, err := s3c.Client.GetObject(&input)
@@ -216,9 +251,12 @@ func (s3c *S3Cache) Get(key *cache.Key) ([]byte, bool, error) {
 func (s3c *S3Cache) Purge(key *cache.Key) error {
 	var err error
 
+	//	add our basepath
+	k := filepath.Join(s3c.Basepath, key.String())
+
 	input := s3.DeleteObjectInput{
 		Bucket: aws.String(s3c.Bucket),
-		Key:    aws.String(key.String()),
+		Key:    aws.String(k),
 	}
 
 	_, err = s3c.Client.DeleteObject(&input)

cache/s3cache/s3cache_test.go

@@ -22,8 +22,8 @@ func TestNew(t *testing.T) {
 		//	test static creds
 		{
 			config: map[string]interface{}{
-				"bucket":                "tegola-test-data",
-				"region":                "us-west-1",
+				"bucket":                os.Getenv("AWS_TEST_BUCKET"),
+				"region":                os.Getenv("AWS_REGION"),
 				"aws_access_key_id":     os.Getenv("AWS_ACCESS_KEY_ID"),
 				"aws_secret_access_key": os.Getenv("AWS_SECRET_ACCESS_KEY"),
 			},
@@ -32,9 +32,9 @@ func TestNew(t *testing.T) {
 		//	test env var creds and max zoom
 		{
 			config: map[string]interface{}{
-				"bucket":   "tegola-test-data",
+				"bucket":   os.Getenv("AWS_TEST_BUCKET"),
 				"max_zoom": 9,
-				"region":   "us-west-1",
+				"region":   os.Getenv("AWS_REGION"),
 			},
 			err: nil,
 		},
@@ -46,7 +46,7 @@ func TestNew(t *testing.T) {
 		//	invalid value for max_zoom
 		{
 			config: map[string]interface{}{
-				"bucket":   "tegola-test-data",
+				"bucket":   os.Getenv("AWS_TEST_BUCKET"),
 				"max_zoom": "foo",
 			},
 			err: fmt.Errorf("max_zoom value needs to be of type int. Value is of type string"),
@@ -82,13 +82,14 @@ func TestSetGetPurge(t *testing.T) {
 	}{
 		{
 			config: map[string]interface{}{
-				"bucket": "tegola-test-data",
-				"region": "us-west-1",
+				"bucket":   os.Getenv("AWS_TEST_BUCKET"),
+				"basepath": "cache",
 			},
 			key: cache.Key{
-				Z: 0,
-				X: 1,
-				Y: 2,
+				MapName: "test-map",
+				Z:       0,
+				X:       1,
+				Y:       2,
 			},
 			expected: []byte{0x53, 0x69, 0x6c, 0x61, 0x73},
 		},