[dataprovider] Added Observability to postgis data provider

* Observability of the postgis pool
* MVTProvider has query time observability on a per map and zoom level
* Provider has query time observability on a per map, per layer and zoom level

atlas/atlas.go

@@ -121,6 +121,7 @@ func (a *Atlas) SeedMapTile(ctx context.Context, m Map, z, x, y uint) error {
 		return defaultAtlas.SeedMapTile(ctx, m, z, x, y)
 	}
 
+	ctx = context.WithValue(ctx, observability.ObserveVarMapName, m.Name)
 	// confirm we have a cache backend
 	if a.cacher == nil {
 		return ErrMissingCache
@@ -258,6 +259,15 @@ func (a *Atlas) SetObservability(o observability.Interface) {
 			a.cacher = o.InstrumentedCache(a.cacher)
 		}
 	}
+	for _, aMap := range a.maps {
+
+		collectors, err := aMap.Collectors("tegola", o.CollectorConfig)
+		if err != nil {
+			log.Printf("failed to register collector for map: %v ignoring", aMap.Name)
+			continue
+		}
+		o.MustRegister(collectors...)
+	}
 }
 
 func (a *Atlas) Observer() observability.Interface {

atlas/layer.go

@@ -2,6 +2,7 @@ package atlas
 
 import (
 	"github.com/go-spatial/geom"
+	"github.com/go-spatial/tegola/observability"
 	"github.com/go-spatial/tegola/provider"
 )
 
@@ -32,3 +33,11 @@ func (l *Layer) MVTName() string {
 
 	return l.ProviderLayerName
 }
+
+func (l Layer) Collectors(prefix string, config func(configKey string) map[string]interface{}) ([]observability.Collector, error) {
+	collect, ok := l.Provider.(observability.Observer)
+	if !ok {
+		return nil, nil
+	}
+	return collect.Collectors(prefix, config)
+}

atlas/map.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/go-spatial/tegola/observability"
+
 	"github.com/golang/protobuf/proto"
 
 	"github.com/go-spatial/geom"
@@ -59,6 +61,8 @@ type Map struct {
 
 	mvtProviderName string
 	mvtProvider     provider.MVTTiler
+
+	observer observability.Interface
 }
 
 // HasMVTProvider indicates if map is a mvt provider based map
@@ -77,6 +81,28 @@ func (m *Map) SetMVTProvider(name string, p provider.MVTTiler) provider.MVTTiler
 	return p
 }
 
+func (m Map) Collectors(prefix string, config func(configKey string) map[string]interface{}) ([]observability.Collector, error) {
+	if m.mvtProviderName != "" {
+		collect, ok := m.mvtProvider.(observability.Observer)
+		if !ok {
+			return nil, nil
+		}
+		return collect.Collectors(prefix, config)
+	}
+	// not an mvtProvider, so need to ask each layer instead
+	var collection []observability.Collector
+	for i := range m.Layers {
+		aCollection, err := m.Layers[i].Collectors(prefix, config)
+		if err != nil {
+			return nil, err
+		}
+		if len(aCollection) != 0 {
+			collection = append(collection, aCollection...)
+		}
+	}
+	return collection, nil
+}
+
 // AddDebugLayers returns a copy of a Map with the debug layers appended to the layer list
 func (m Map) AddDebugLayers() Map {
 	// can not modify the layers of an mvt provider based map

internal/observer/null.go

@@ -3,6 +3,8 @@ package observer
 import (
 	"net/http"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/go-spatial/tegola/cache"
 )
 
@@ -11,8 +13,10 @@ type Null struct{}
 // Handler does nothing.
 func (Null) Handler(string) http.Handler { return nil }
 
-func (Null) Init()     {}
-func (Null) Shutdown() {}
+func (Null) Init()                                           {}
+func (Null) Shutdown()                                       {}
+func (Null) MustRegister(_ ...prometheus.Collector)          {}
+func (Null) CollectorConfig(_ string) map[string]interface{} { return make(map[string]interface{}) }
 
 // InstrumentedAPIHttpHandler does not do anything and just returns the handler
 func (Null) InstrumentedAPIHttpHandler(_, _ string, handler http.Handler) http.Handler {

observability/observability.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"sort"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	tegolaCache "github.com/go-spatial/tegola/cache"
 
 	"github.com/go-spatial/tegola/dict"
@@ -18,6 +20,8 @@ const (
 	ObserveVarTileZ     = ":z"
 )
 
+type Collector = prometheus.Collector
+
 // ErrObserverAlreadyExists is returned if an observer try to register to an observer type that has already registered
 type ErrObserverAlreadyExists string
 
@@ -52,6 +56,12 @@ type Interface interface {
 	// Shutdown should shutdown any subsystems that Init setup
 	Shutdown()
 
+	// MustRegister will register any custom collectors. It will panic if there is an error
+	MustRegister(collectors ...Collector)
+
+	// CollectorConfig returns the config for a given key, or nil, if the key does not exist
+	CollectorConfig(key string) map[string]interface{}
+
 	APIObserver
 	ViewerObserver
 	CacheObserver
@@ -79,6 +89,13 @@ type Cache interface {
 	IsObserver() bool
 }
 
+// Observer is able to be observed via the collectors it provides
+type Observer interface {
+	// Collectors should return a set of collectors that will be registered by the default observability provider,
+	// to get the configuration; use the provided function and your config key.
+	Collectors(prefix string, config func(configKey string) map[string]interface{}) ([]Collector, error)
+}
+
 // InstrumentAPIHandler is a convenience  function
 func InstrumentAPIHandler(method, route string, observer APIObserver, handler http.Handler) (string, string, http.Handler) {
 	if observer == nil {
@@ -158,3 +175,20 @@ func Cleanup() {
 		o.CleanUp()
 	}
 }
+
+func LabelForObserveVar(key string) string {
+	switch key {
+	case ObserveVarMapName:
+		return "map_name"
+	case ObserveVarLayerName:
+		return "layer_name"
+	case ObserveVarTileX:
+		return "x"
+	case ObserveVarTileY:
+		return "y"
+	case ObserveVarTileZ:
+		return "z"
+	default:
+		return ""
+	}
+}

observability/prometheus/README.md

@@ -3,7 +3,7 @@
 The Prometheus Observability provider manages the collection of various
 metrics for Tegola's various subsystems.
 
-The connection between Tegola and Prometheus is configured in the Tegola config
+The connection between Tegola and Prometheus is configured in the config
 file (`tegola.toml`.) An example of minimum configuration :
 
 ```toml
@@ -18,7 +18,7 @@ The metrics will be exposed on the `/metrics` end point.
 ### Configuration Properties
 
 - `type` (string): [Required] the type of the observer, must be "`prometheus`".
-- `variables` (array of strings): [Optional] A set of tokens that should be replaced in the url for HTTP or additional labels for cache providers. Currently, supported tokenks are:
+- `variables` (array of strings): [Optional] A set of tokens that should be replaced in the url for HTTP or additional labels for cache providers. Currently, supported tokens are:
   * `:map_name` [Default]
   * `:layer_name` [Default]
   * `:z` [Default]
@@ -38,7 +38,7 @@ A gauge with build information from the running command
 
 ###### Labels
 
-* branch - the git branch this binary was built on if set
+* branch - if set, the git branch of this built binary
 * revision - the git short revision number
 * version - the set version number
 * command - the running command
@@ -142,7 +142,7 @@ A counter of the number of tile hits
 ###### labels
 
 * sub_command is the cache command which is one of "get","set", or "purge"
-* layer_name is an optional label, that is the layer_name ; this is only presetn if configured via `variables` config option.
+* layer_name is an optional label, that is the layer_name ; this is only present if configured via `variables` config option.
 * map_name is an optional label, that is the map_name; this is only present if configured via `variables` config option.
 * z is an optional label, that is the z coordinate; this is only present if configured via `variables` config option.
 * x is an optional label, that is the x coordinate; this is only present if configured via `variables` config option.
@@ -155,7 +155,7 @@ A counter of the number of tile hits
 ###### labels
 
 * sub_command is the cache command which is one of "get","set", or "purge"
-* layer_name is an optional label, that is the layer_name ; this is only presetn if configured via `variables` config option.
+* layer_name is an optional label, that is the layer_name ; this is only present if configured via `variables` config option.
 * map_name is an optional label, that is the map_name; this is only present if configured via `variables` config option.
 * z is an optional label, that is the z coordinate; this is only present if configured via `variables` config option.
 * x is an optional label, that is the x coordinate; this is only present if configured via `variables` config option.
@@ -172,7 +172,7 @@ As part of a histogram include the support tags:
 ###### labels
 
 * sub_command is the cache command which is one of "get","set", or "purge"
-* layer_name is an optional label, that is the layer_name ; this is only presetn if configured via `variables` config option.
+* layer_name is an optional label, that is the layer_name ; this is only present if configured via `variables` config option.
 * map_name is an optional label, that is the map_name; this is only present if configured via `variables` config option.
 * z is an optional label, that is the z coordinate; this is only present if configured via `variables` config option.
 * x is an optional label, that is the x coordinate; this is only present if configured via `variables` config option.
@@ -190,13 +190,44 @@ As part of a histogram include the support tags:
 ###### labels
 
 * sub_command is the cache command which is one of "get","set", or "purge"
-* layer_name is an optional label, that is the layer_name ; this is only presetn if configured via `variables` config option.
+* layer_name is an optional label, that is the layer_name ; this is only present if configured via `variables` config option.
 * map_name is an optional label, that is the map_name; this is only present if configured via `variables` config option.
 * z is an optional label, that is the z coordinate; this is only present if configured via `variables` config option.
 * x is an optional label, that is the x coordinate; this is only present if configured via `variables` config option.
 * y is an optional label, that is the y coordinate; this is only present if configured via `variables` config option.
 * le is the buckets in bytes
 
+
+#### tegola data provider postgres
+
+##### tegola_postgres_max_connections
+
+The max number of postgres connections in the connection pool.
+
+##### tegola_postgres_current_connections
+
+The current number of postgres connections in the connection pool.
+
+##### tegola_postgres_available_connections
+
+The current number of available postgres connections in the connection pool.
+
+##### tegola_mvt_provider_sql_query_seconds
+
+Labels: "map_name", and "z"
+Buckets: .1 second, 1 second, 5 seconds, and 20+ seconds
+
+A histogram of the query time for the SQLs for the mvt provider
+
+##### tegola_provider_sql_query_seconds
+
+Labels: "map_name", "layer_name", and "z"
+Buckets: .1 second, 1 second, 5 seconds, and 20+ seconds
+
+A histogram of the query time for the SQLs for the provider
+
+
+
 #### go runtime information
 
 ##### go_gc_duration_seconds
@@ -233,7 +264,7 @@ The fraction of this program's available CPU time used by the GC since the progr
 
 ##### go_memstats_gc_sys_bytes
 
-Number of bytes used for garbage collection system metadata.
+Number of bytes used for garbage-collection-system metadata.
 
 ##### go_memstats_heap_alloc_bytes
 

observability/prometheus/cache.go

@@ -88,18 +88,9 @@ func newCache(registry prometheus.Registerer, prefix string, observeVars []strin
 // labelNames returns the label name based on the configured observeVars and "sub_command"
 func (co *cache) labelNames() (names []string) {
 	names = []string{"sub_command"}
-	for _, keyName := range co.observeVars {
-		switch keyName {
-		case observability.ObserveVarMapName:
-			names = append(names, "map_name")
-		case observability.ObserveVarLayerName:
-			names = append(names, "layer_name")
-		case observability.ObserveVarTileX:
-			names = append(names, "x")
-		case observability.ObserveVarTileY:
-			names = append(names, "y")
-		case observability.ObserveVarTileZ:
-			names = append(names, "z")
+	for _, key := range co.observeVars {
+		if name := observability.LabelForObserveVar(key); name != "" {
+			names = append(names, name)
 		}
 	}
 	return names

observability/prometheus/prometheus.go

@@ -156,6 +156,14 @@ func (obs *observer) Shutdown() {
 	cleanUpFunctionsLck.Unlock()
 }
 
+func (obs *observer) MustRegister(collectors ...observability.Collector) {
+	obs.registry.MustRegister(collectors...)
+}
+
+func (_ *observer) CollectorConfig(_ string) map[string]interface{} {
+	return make(map[string]interface{})
+}
+
 func (obs *observer) PublishBuildInfo() { obs.publishedBuildInfo.Do(PublishBuildInfo) }
 
 func (obs *observer) InstrumentedAPIHttpHandler(method, route string, next http.Handler) http.Handler {
@@ -200,7 +208,10 @@ var (
 func cleanUp() {
 	cleanUpFunctionsLck.Lock()
 	for i := range cleanUpFunctions {
-		cleanUpFunctions[i]()
+		if cleanUpFunctions[i] != nil {
+			cleanUpFunctions[i]()
+			cleanUpFunctions[i] = nil
+		}
 	}
 	cleanUpFunctions = cleanUpFunctions[:0]
 	cleanUpFunctionsLck.Unlock()