chore: add CLI invokation telemetry (coder#7589)

cli/clibase/cmd.go

@@ -145,6 +145,16 @@ func (c *Cmd) FullUsage() string {
 	return strings.Join(uses, " ")
 }
 
+// FullOptions returns the options of the command and its parents.
+func (c *Cmd) FullOptions() OptionSet {
+	var opts OptionSet
+	if c.Parent != nil {
+		opts = append(opts, c.Parent.FullOptions()...)
+	}
+	opts = append(opts, c.Options...)
+	return opts
+}
+
 // Invoke creates a new invocation of the command, with
 // stdio discarded.
 //

cli/root.go

@@ -2,6 +2,8 @@ package cli
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -33,6 +35,7 @@ import (
 	"github.com/coder/coder/cli/config"
 	"github.com/coder/coder/coderd"
 	"github.com/coder/coder/coderd/gitauth"
+	"github.com/coder/coder/coderd/telemetry"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/codersdk/agentsdk"
 )
@@ -425,6 +428,24 @@ type RootCmd struct {
 	noFeatureWarning bool
 }
 
+func telemetryInvocation(i *clibase.Invocation) telemetry.CLIInvocation {
+	var topts []telemetry.CLIOption
+	for _, opt := range i.Command.FullOptions() {
+		if opt.ValueSource == clibase.ValueSourceNone || opt.ValueSource == clibase.ValueSourceDefault {
+			continue
+		}
+		topts = append(topts, telemetry.CLIOption{
+			Name:        opt.Name,
+			ValueSource: string(opt.ValueSource),
+		})
+	}
+	return telemetry.CLIInvocation{
+		Command:   i.Command.FullName(),
+		Options:   topts,
+		InvokedAt: time.Now(),
+	}
+}
+
 // InitClient sets client to a new client.
 // It reads from global configuration files if flags are not set.
 func (r *RootCmd) InitClient(client *codersdk.Client) clibase.MiddlewareFunc {
@@ -465,7 +486,18 @@ func (r *RootCmd) InitClient(client *codersdk.Client) clibase.MiddlewareFunc {
 				}
 			}
 
-			err = r.setClient(client, r.clientURL)
+			telemInv := telemetryInvocation(i)
+			byt, err := json.Marshal(telemInv)
+			if err != nil {
+				// Should be impossible
+				panic(err)
+			}
+			err = r.setClient(
+				client, r.clientURL,
+				append(r.header, codersdk.CLITelemetryHeader+"="+
+					base64.StdEncoding.EncodeToString(byt),
+				),
+			)
 			if err != nil {
 				return err
 			}
@@ -512,12 +544,12 @@ func (r *RootCmd) InitClient(client *codersdk.Client) clibase.MiddlewareFunc {
 	}
 }
 
-func (r *RootCmd) setClient(client *codersdk.Client, serverURL *url.URL) error {
+func (*RootCmd) setClient(client *codersdk.Client, serverURL *url.URL, headers []string) error {
 	transport := &headerTransport{
 		transport: http.DefaultTransport,
 		header:    http.Header{},
 	}
-	for _, header := range r.header {
+	for _, header := range headers {
 		parts := strings.SplitN(header, "=", 2)
 		if len(parts) < 2 {
 			return xerrors.Errorf("split header %q had less than two parts", header)
@@ -533,7 +565,7 @@ func (r *RootCmd) setClient(client *codersdk.Client, serverURL *url.URL) error {
 
 func (r *RootCmd) createUnauthenticatedClient(serverURL *url.URL) (*codersdk.Client, error) {
 	var client codersdk.Client
-	err := r.setClient(&client, serverURL)
+	err := r.setClient(&client, serverURL, r.header)
 	return &client, err
 }
 

cli/vscodessh.go

@@ -83,7 +83,7 @@ func (r *RootCmd) vscodeSSH() *clibase.Cmd {
 			client.SetSessionToken(string(sessionToken))
 
 			// This adds custom headers to the request!
-			err = r.setClient(client, serverURL)
+			err = r.setClient(client, serverURL, r.header)
 			if err != nil {
 				return xerrors.Errorf("set client: %w", err)
 			}

coderd/coderd.go

@@ -465,6 +465,7 @@ func New(options *Options) *API {
 			// Specific routes can specify different limits, but every rate
 			// limit must be configurable by the admin.
 			apiRateLimiter,
+			httpmw.ReportCLITelemetry(api.Logger, options.Telemetry),
 		)
 		r.Get("/", apiRoot)
 		// All CSP errors will be logged

coderd/httpmw/clitelemetry.go

@@ -0,0 +1,80 @@
+package httpmw
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"sync"
+	"time"
+
+	"tailscale.com/tstime/rate"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/coderd/telemetry"
+	"github.com/coder/coder/codersdk"
+)
+
+func ReportCLITelemetry(log slog.Logger, rep telemetry.Reporter) func(http.Handler) http.Handler {
+	var (
+		mu sync.Mutex
+
+		// We send telemetry at most once per minute.
+		limiter = rate.NewLimiter(rate.Every(time.Minute), 1)
+		queue   []telemetry.CLIInvocation
+	)
+
+	log = log.Named("cli-telemetry")
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			// No matter what, we proceed with the request.
+			defer next.ServeHTTP(rw, r)
+
+			payload := r.Header.Get(codersdk.CLITelemetryHeader)
+			if payload == "" {
+				return
+			}
+
+			byt, err := base64.StdEncoding.DecodeString(payload)
+			if err != nil {
+				log.Error(
+					r.Context(),
+					"base64 decode",
+					slog.F("error", err),
+				)
+				return
+			}
+
+			var inv telemetry.CLIInvocation
+			err = json.Unmarshal(byt, &inv)
+			if err != nil {
+				log.Error(
+					r.Context(),
+					"unmarshal header",
+					slog.Error(err),
+				)
+				return
+			}
+
+			// We do expensive work in a goroutine so we don't block the
+			// request.
+			go func() {
+				mu.Lock()
+				defer mu.Unlock()
+
+				queue = append(queue, inv)
+				if !limiter.Allow() && len(queue) < 1024 {
+					return
+				}
+				rep.Report(&telemetry.Snapshot{
+					CLIInvocations: queue,
+				})
+				log.Debug(
+					r.Context(),
+					"report sent", slog.F("count", len(queue)),
+				)
+				queue = queue[:0]
+			}()
+		})
+	}
+}

coderd/telemetry/telemetry.go

@@ -701,6 +701,7 @@ type Snapshot struct {
 	WorkspaceBuilds           []WorkspaceBuild            `json:"workspace_build"`
 	WorkspaceResources        []WorkspaceResource         `json:"workspace_resources"`
 	WorkspaceResourceMetadata []WorkspaceResourceMetadata `json:"workspace_resource_metadata"`
+	CLIInvocations            []CLIInvocation             `json:"cli_invocations"`
 }
 
 // Deployment contains information about the host running Coder.
@@ -876,6 +877,18 @@ type License struct {
 	UUID       uuid.UUID `json:"uuid"`
 }
 
+type CLIOption struct {
+	Name        string `json:"name"`
+	ValueSource string `json:"value_source"`
+}
+
+type CLIInvocation struct {
+	Command string      `json:"command"`
+	Options []CLIOption `json:"options"`
+	// InvokedAt is provided for deduplication purposes.
+	InvokedAt time.Time `json:"invoked_at"`
+}
+
 type noopReporter struct{}
 
 func (*noopReporter) Report(_ *Snapshot) {}

codersdk/client.go

@@ -61,6 +61,16 @@ const (
 	// Only owners can bypass rate limits. This is typically used for scale testing.
 	// nolint: gosec
 	BypassRatelimitHeader = "X-Coder-Bypass-Ratelimit"
+
+	// Note: the use of X- prefix is deprecated, and we should eventually remove
+	// it from BypassRatelimitHeader.
+	//
+	// See: https://datatracker.ietf.org/doc/html/rfc6648.
+
+	// CLITelemetryHeader contains a base64-encoded representation of the CLI
+	// command that was invoked to produce the request. It is for internal use
+	// only.
+	CLITelemetryHeader = "Coder-CLI-Telemetry"
 )
 
 // loggableMimeTypes is a list of MIME types that are safe to log
@@ -179,15 +189,6 @@ func (c *Client) Request(ctx context.Context, method, path string, body interfac
 		return nil, xerrors.Errorf("create request: %w", err)
 	}
 
-	if c.PlainLogger != nil {
-		out, err := httputil.DumpRequest(req, c.LogBodies)
-		if err != nil {
-			return nil, xerrors.Errorf("dump request: %w", err)
-		}
-		out = prefixLines([]byte("http --> "), out)
-		_, _ = c.PlainLogger.Write(out)
-	}
-
 	tokenHeader := c.SessionTokenHeader
 	if tokenHeader == "" {
 		tokenHeader = SessionTokenHeader
@@ -221,6 +222,18 @@ func (c *Client) Request(ctx context.Context, method, path string, body interfac
 	})
 
 	resp, err := c.HTTPClient.Do(req)
+
+	// We log after sending the request because the HTTP Transport may modify
+	// the request within Do, e.g. by adding headers.
+	if resp != nil && c.PlainLogger != nil {
+		out, err := httputil.DumpRequest(resp.Request, c.LogBodies)
+		if err != nil {
+			return nil, xerrors.Errorf("dump request: %w", err)
+		}
+		out = prefixLines([]byte("http --> "), out)
+		_, _ = c.PlainLogger.Write(out)
+	}
+
 	if err != nil {
 		return nil, err
 	}