add alert error code reporting as well

alert.go

@@ -356,9 +356,10 @@ func newAlertNode(et *ExecutingTask, n *pipeline.AlertNode, d NodeDiagnostic) (a
 
 	for _, p := range n.HTTPPostHandlers {
 		c := httppost.HandlerConfig{
-			URL:      p.URL,
-			Endpoint: p.Endpoint,
-			Headers:  p.Headers,
+			URL:             p.URL,
+			Endpoint:        p.Endpoint,
+			Headers:         p.Headers,
+			CaptureResponse: p.CaptureResponseFlag,
 		}
 		h := et.tm.HTTPPostService.Handler(c, ctx...)
 		an.handlers = append(an.handlers, h)

http_post.go

@@ -159,7 +159,6 @@ func (n *HTTPPostNode) doPost(row *models.Row) int {
 }
 
 func (n *HTTPPostNode) postRow(row *models.Row) (*http.Response, error) {
-
 	body := n.bp.Get()
 	defer n.bp.Put(body)
 

pipeline/alert.go

@@ -507,6 +507,24 @@ func (a *AlertNode) Post(urls ...string) *AlertHTTPPostHandler {
 	return post
 }
 
+// tick:embedded:AlertNode.Post
+type AlertHTTPPostHandler struct {
+	*AlertNode
+
+	// The POST URL.
+	// tick:ignore
+	URL string
+
+	// Name of the endpoint to be used, as is defined in the configuration file
+	Endpoint string
+
+	// tick:ignore
+	Headers map[string]string `tick:"Header"`
+
+	// tick:ignore
+	CaptureResponseFlag bool `tick:"CaptureResponse"`
+}
+
 // Set a header key and value on the post request.
 // Setting the Authenticate header is not allowed from within TICKscript,
 // please use the configuration file to specify sensitive headers.
@@ -527,19 +545,12 @@ func (a *AlertHTTPPostHandler) Header(k, v string) *AlertHTTPPostHandler {
 	return a
 }
 
-// tick:embedded:AlertNode.Post
-type AlertHTTPPostHandler struct {
-	*AlertNode
-
-	// The POST URL.
-	// tick:ignore
-	URL string
-
-	// Name of the endpoint to be used, as is defined in the configuration file
-	Endpoint string
-
-	// tick:ignore
-	Headers map[string]string `tick:"Header"`
+// CaptureResponse indicates that the HTTP response should be read and logged if
+// the status code was not an 2xx code.
+// tick:property
+func (a *AlertHTTPPostHandler) CaptureResponse() *AlertHTTPPostHandler {
+	a.CaptureResponseFlag = true
+	return a
 }
 
 func (a *AlertHTTPPostHandler) validate() error {

services/diagnostic/handlers.go

@@ -648,8 +648,13 @@ type HTTPPostHandler struct {
 	l *klog.Logger
 }
 
-func (h *HTTPPostHandler) Error(msg string, err error) {
-	h.l.Error(msg, klog.Error(err))
+func (h *HTTPPostHandler) Error(msg string, err error, ctx ...keyvalue.T) {
+	fields := make([]klog.Field, len(ctx)+1)
+	fields[0] = klog.Error(err)
+	for i, kv := range ctx {
+		fields[i+1] = klog.String(kv.Key, kv.Value)
+	}
+	h.l.Error(msg, fields...)
 }
 
 func (h *HTTPPostHandler) WithContext(ctx ...keyvalue.T) httppost.Diagnostic {

services/httppost/service.go

@@ -5,7 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"strconv"
 	"sync"
 	"text/template"
 
@@ -17,7 +19,7 @@ import (
 
 type Diagnostic interface {
 	WithContext(ctx ...keyvalue.T) Diagnostic
-	Error(msg string, err error)
+	Error(msg string, err error, ctx ...keyvalue.T)
 }
 
 // Only one of name and url should be non-empty
@@ -228,9 +230,10 @@ func (s *Service) Test(options interface{}) error {
 }
 
 type HandlerConfig struct {
-	URL      string            `mapstructure:"url"`
-	Endpoint string            `mapstructure:"endpoint"`
-	Headers  map[string]string `mapstructure:"headers"`
+	URL             string            `mapstructure:"url"`
+	Endpoint        string            `mapstructure:"endpoint"`
+	Headers         map[string]string `mapstructure:"headers"`
+	CaptureResponse bool              `mapstructure:"capture-response"`
 }
 
 type handler struct {
@@ -240,6 +243,8 @@ type handler struct {
 	endpoint *Endpoint
 	headers  map[string]string
 
+	captureResponse bool
+
 	diag Diagnostic
 }
 
@@ -249,11 +254,12 @@ func (s *Service) Handler(c HandlerConfig, ctx ...keyvalue.T) alert.Handler {
 		e = NewEndpoint(c.URL, nil, BasicAuth{}, nil, nil)
 	}
 	return &handler{
-		s:        s,
-		bp:       bufpool.New(),
-		endpoint: e,
-		diag:     s.diag.WithContext(ctx...),
-		headers:  c.Headers,
+		s:               s,
+		bp:              bufpool.New(),
+		endpoint:        e,
+		diag:            s.diag.WithContext(ctx...),
+		headers:         c.Headers,
+		captureResponse: c.CaptureResponse,
 	}
 }
 
@@ -310,5 +316,21 @@ func (h *handler) Handle(event alert.Event) {
 		h.diag.Error("failed to POST alert data", err)
 		return
 	}
-	resp.Body.Close()
+	defer resp.Body.Close()
+
+	if resp.StatusCode/100 != 2 {
+		var err error
+		if h.captureResponse {
+			var body []byte
+			body, err = ioutil.ReadAll(resp.Body)
+			if err == nil {
+				// Use the body content as the error
+				err = errors.New(string(body))
+			}
+		} else {
+			err = errors.New("unknown error, use .captureResponse() to capture the HTTP response")
+		}
+		h.diag.Error("POST returned non 2xx status code", err, keyvalue.KV("code", strconv.Itoa(resp.StatusCode)))
+	}
+
 }