CDN purge (newrelic#622)

* feat: Fastly purge WIP

* feat: support several keys

* refactor: proper error handling

* feat: purge CDN

* feat: purge script

* feat: trigger CDN purge from GHA

* fix: purge cdn timeout

* fix: timeout CDN

.github/workflows/release.yml

@@ -146,6 +146,13 @@ jobs:
           disable_lock: true
           aws_region: ${{ env.AWS_REGION }}
 
+  purge-cdn:
+    - name: Purge CDN
+      needs: [ publishing-to-s3-windows, publishing-to-s3-linux, publishing-to-s3-macos ]
+      env:
+        FASTLY_KEY: ${{secrets.FASTLY_KEY }}
+      run: tools/cdn-purge/fastly-purge.sh
+
   publish-docker-images:
     name: Create versioned and latest images from RC
     runs-on: ubuntu-20.04

.github/workflows/release_staged.yml

@@ -166,6 +166,13 @@ jobs:
           disable_lock: true
           aws_region: ${{ env.AWS_REGION }}
 
+  purge-cdn:
+    - name: Purge CDN
+      needs: [ publishing-to-s3-windows, publishing-to-s3-linux, publishing-to-s3-macos ]
+      env:
+        FASTLY_KEY: ${{secrets.FASTLY_KEY }}
+      run: tools/cdn-purge/fastly-purge.sh
+
   publish-docker-images:
     if: ${{ github.event.inputs.assetsType == 'all' || github.event.inputs.assetsType == 'docker' }}
     name: Create versioned and latest images from RC

tools/cdn-purge/.gitignore

@@ -0,0 +1,2 @@
+vendor/
+

tools/cdn-purge/fastly-purge.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// Usage:
+// go run fastly-purge.go -v
+//
+// Similar shell counterpart:
+// for i in {1..5}; do
+//	echo \$i;
+//	aws s3api head-object --bucket nr-downloads-main --key infrastructure_agent/linux/yum/el/7/x86_64/repodata/primary.sqlite.bz2
+//		|/bin/grep ReplicationStatus
+//		|/bin/grep COMPLETED
+//		&& /usr/bin/curl -i -X POST -H \"Fastly-Key:\${FASTLY_KEY}\" https://api.fastly.com/service/2RMeBJ1ZTGnNJYvrWMgQhk/purge_all
+//		&& break ;
+//	/bin/sleep 60s;
+//	if [ \$i -ge 5 ]; then
+//		/usr/bin/curl -i -X POST -H \"Fastly-Key:\${FASTLY_KEY}\" https://api.fastly.com/service/2RMeBJ1ZTGnNJYvrWMgQhk/purge_all;
+//	fi;
+// done
+
+type result struct {
+	output s3.GetObjectOutput
+	err    error
+}
+
+const (
+	defaultBucket = "nr-downloads-ohai-staging"
+	defaultRegion = "us-east-1"
+	// more keys could be added if issues arise
+	defaultKeys    = "/infrastructure_agent/linux/apt/dists/focal/main/binary-amd64/Packages.bz2,"
+	fastlyPurgeURL = "https://api.fastly.com/service/2RMeBJ1ZTGnNJYvrWMgQhk/purge_all"
+)
+
+var bucket, region, keysStr, fastlyKey string
+var timeoutS3, timeoutCDN time.Duration
+var attempts int
+var verbose bool
+
+func init() {
+	flag.BoolVar(&verbose, "v", false, "Verbose output.")
+	flag.StringVar(&bucket, "b", defaultBucket, "Bucket name.")
+	flag.StringVar(&region, "r", defaultRegion, "Region name.")
+	flag.StringVar(&keysStr, "k", defaultKeys, "Keys separated by comma.")
+	flag.IntVar(&attempts, "a", 5, "Retry attempts per key.")
+	flag.DurationVar(&timeoutS3, "t", 10*time.Second, "Timeout to fetch an S3 object.")
+	flag.DurationVar(&timeoutCDN, "c", 30*time.Second, "Timeout to request CDN purge.")
+}
+
+func main() {
+	flag.Parse()
+
+	var ok bool
+	fastlyKey, ok = os.LookupEnv("FASTLY_KEY")
+	if !ok {
+		logInfo("missing required env-var FASTLY_KEY")
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+
+	sess := session.Must(session.NewSession())
+	cl := s3.New(sess, aws.NewConfig().WithRegion(region))
+
+	keys := strings.Split(keysStr, ",")
+	for _, key := range keys {
+		if key != "" {
+			if err := waitForKeyReplication(ctx, key, cl, attempts); err != nil {
+				logInfo("unsucessful replication, error: %v", err)
+				os.Exit(1)
+			}
+		}
+	}
+
+	if err := purgeCDN(ctx); err != nil {
+		logInfo("cannot purge CDN, error: %v", err)
+	}
+}
+
+// waitForKeyReplication returns nil if key was successfully replicated or is not set for replication
+func waitForKeyReplication(ctx context.Context, key string, cl *s3.S3, triesLeft int) error {
+	inputGetObj := s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &key,
+	}
+
+	replicated := false
+	for {
+		if replicated || triesLeft <= 0 {
+			break
+		}
+		triesLeft--
+
+		var ctxT = ctx
+		var cancelFn func()
+		if timeoutS3 > 0 {
+			ctxT, cancelFn = context.WithTimeout(ctx, timeoutS3)
+		}
+		if cancelFn != nil {
+			defer cancelFn()
+		}
+
+		resC := make(chan result)
+		go func(*s3.S3) {
+			o, err := cl.GetObjectWithContext(ctxT, &inputGetObj)
+			if err != nil {
+				resC <- result{err: err}
+			}
+			resC <- result{output: *o}
+		}(cl)
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("execution terminated, msg: %v", ctx.Err())
+
+		case res := <-resC:
+			if res.err != nil {
+				return fmt.Errorf("cannot get s3 object, key: %s, error: %v", key, res.err)
+			}
+
+			logDebug("key: %s, attempt: %d, object: %+v", key, attempts-triesLeft, res.output)
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/replication-status.html
+			// aws s3api head-object --bucket foo --key "bar/..." |grep ReplicationStatus
+			if res.output.ReplicationStatus == nil || *res.output.ReplicationStatus == s3.ReplicationStatusComplete {
+				replicated = true
+			}
+		}
+	}
+
+	if triesLeft <= 0 {
+		return fmt.Errorf("maximum attempts for key: %v", key)
+	}
+
+	return nil
+}
+
+func purgeCDN(ctx context.Context) error {
+	var ctxT = ctx
+	var cancelFn func()
+	if timeoutCDN > 0 {
+		ctxT, cancelFn = context.WithTimeout(ctx, timeoutCDN)
+	}
+	if cancelFn != nil {
+		defer cancelFn()
+	}
+
+	req, err := http.NewRequestWithContext(ctxT, http.MethodPost, fastlyPurgeURL, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Fastly-Key", fastlyKey)
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if res.StatusCode < 200 || res.StatusCode >= 400 {
+		return fmt.Errorf("unexpected Fastly status: %s", res.Status)
+	}
+
+	return nil
+}
+
+func logInfo(format string, v ...interface{}) {
+	log.Printf(format, v...)
+}
+
+func logDebug(format string, v ...interface{}) {
+	if verbose {
+		log.Printf(format, v...)
+	}
+}

tools/cdn-purge/fastly-purge.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd tools/cdn-purge
+go mod vendor
+FASTLY_KEY=${FASTLY_KEY} go run fastly-purge.go -v

tools/cdn-purge/go.mod

@@ -0,0 +1,5 @@
+module github.com/newrelic/infrastructure-agent/tools/cdn-purge
+
+go 1.16
+
+require github.com/aws/aws-sdk-go v1.39.0

tools/cdn-purge/go.sum

@@ -0,0 +1,27 @@
+github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo=
+github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=