Add support for custom registries, registry auth

Add auth support for registries through a config file with the same
format as used by Docker and Kubernetes. The implementation uses the
existing auth implementations for select registries as fallback,
assuming all other repositories allow anonymous access or that auth is
configured.

For now, there's only partial support for HTTP-based registries as there
is no nice way to configure what registries use HTTP. Additional work
could look at how the Docker CLI handles this.

With this change, images with IPs (IPv4 or IPv6) in their name should be
supported. There's currently only partial support for such images in the
UI.

.gitignore

@@ -6,3 +6,4 @@
 /*.sqlite*
 /*.boltdb
 /docker.sock
+/integration/zot/htpasswd

CONTRIBUTING.md

@@ -129,3 +129,31 @@ as the Docker host rather then the one specified in `.env-docker`.
 ```shell
 go run tools/sockproxy/*.go -p 3000 docker.sock
 ```
+
+### Testing custom registry
+
+To test custom registries and authentication, Zot can be used.
+
+```shell
+# Create a htpasswd for zot
+htpasswd -bBn username password > integration/zot/htpasswd
+```
+
+```shell
+docker run --rm -it -p 9090:9090 --volume "$PWD/integration/zot:/etc/zot:ro" ghcr.io/project-zot/zot-linux-arm64
+```
+
+Note that Zot's UI doesn't work on Safari ATM - you will just be logged out if
+you log in.
+
+Run an image using zot instead.
+
+```shell
+docker run --rm -it localhost:9090/alpine
+```
+
+Start Cupdate targeting Docker, specifying the auth file.
+
+```shell
+export CUPDATE_REGISTRY_SECRETS="integration/zot/docker-basic-auth.json"
+```

cmd/cupdate/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net/http"
@@ -86,6 +87,10 @@ type Config struct {
 		Target   string `env:"TARGET"`
 		Insecure bool   `env:"INSECURE"`
 	} `envPrefix:"OTEL_"`
+
+	Registry struct {
+		Secrets string `env:"SECRETS"`
+	} `envPrefix:"REGISTRY_"`
 }
 
 func main() {
@@ -118,6 +123,38 @@ func main() {
 
 	slog.Debug("Parsed config", slog.Any("config", config))
 
+	registryAuth := httputil.NewAuthMux()
+	if config.Registry.Secrets != "" {
+		file, err := os.Open(config.Registry.Secrets)
+		if err != nil {
+			slog.Error("Failed to read registry secrets", slog.Any("error", err))
+			os.Exit(1)
+		}
+
+		var dockerConfig *docker.ConfigFile
+		err = json.NewDecoder(file).Decode(&dockerConfig)
+		file.Close()
+		if err != nil {
+			slog.Error("Failed to parse registry secrets", slog.Any("error", err))
+			os.Exit(1)
+		}
+
+		for k, v := range dockerConfig.HttpHeaders {
+			registryAuth.SetHeader(k, v)
+		}
+
+		for pattern, auth := range dockerConfig.Auths {
+			if auth.Auth == "" {
+				registryAuth.Handle(pattern, httputil.BasicAuthHandler{
+					Username: auth.Username,
+					Password: auth.Password,
+				})
+			} else {
+				registryAuth.Handle(pattern, httputil.BearerToken(auth.Auth))
+			}
+		}
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 
 	if config.OTEL.Target != "" {
@@ -249,7 +286,7 @@ func main() {
 		httpClient.UserAgent = config.HTTP.UserAgent
 		prometheus.DefaultRegisterer.MustRegister(httpClient)
 
-		worker := worker.New(httpClient, writeStore)
+		worker := worker.New(httpClient, writeStore, registryAuth)
 		prometheus.DefaultRegisterer.MustRegister(worker)
 
 		gauge := prometheus.NewGauge(prometheus.GaugeOpts{

docs/config.md

@@ -29,3 +29,4 @@ done using environment variables.
 | `CUPDATE_DOCKER_INCLUDE_ALL_CONTAINERS`   | Whether or not to include containers in any state, not just running containers.                                                                         | `false`                         |
 | `CUPDATE_OTEL_TARGET`                     | Target URL to an Open Telemetry GRPC ingest endpoint.                                                                                                   | Required to use Open Telemetry. |
 | `CUPDATE_OTEL_INSECURE`                   | Disable client transport security for the Open Telemetry GRPC connection.                                                                               | `false`                         |
+| `CUPDATE_REGISTRY_SECRETS`                | Path to a JSON file containing registry secrets. See Docker's config.json and Kubernetes' `imagePullSecrets`.                                           | None                            |

integration/zot/config.json

@@ -0,0 +1,30 @@
+{
+  "distSpecVersion": "1.0.1",
+  "storage": {
+    "rootDirectory": "/tmp/zot/storage"
+  },
+  "log": {
+    "level": "debug"
+  },
+  "http": {
+    "address": "0.0.0.0",
+    "port": "9090",
+    "auth": {
+      "htpasswd": {
+        "path": "/etc/zot/htpasswd"
+      },
+      "apikey": true
+    }
+  },
+  "extensions": {
+    "sync": {
+      "enable": true,
+      "registries": [
+        {
+          "urls": ["https://docker.io/library"],
+          "onDemand": true
+        }
+      ]
+    }
+  }
+}

integration/zot/docker-basic-auth.json

@@ -0,0 +1,8 @@
+{
+  "auths": {
+    "http://localhost:9090/v2/": {
+      "username": "username",
+      "password": "password"
+    }
+  }
+}

internal/dockerhub/client.go

@@ -7,19 +7,18 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/AlexGustafsson/cupdate/internal/httputil"
 	"github.com/AlexGustafsson/cupdate/internal/oci"
 )
 
-var _ oci.Authorizer = (*Client)(nil)
-
 type Client struct {
 	Client *httputil.Client
 }
 
-func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (string, error) {
+func (c *Client) GetRegistryToken(ctx context.Context, repository string) (string, error) {
 	// TODO: Registries expose the realm and scheme via Www-Authenticate if 403
 	// is given
 	u, err := url.Parse("https://auth.docker.io/token?service=registry.docker.io")
@@ -28,7 +27,7 @@ func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (str
 	}
 
 	query := u.Query()
-	query.Set("scope", fmt.Sprintf("repository:%s:pull", image.Path))
+	query.Set("scope", fmt.Sprintf("repository:%s:pull", repository))
 	u.RawQuery = query.Encode()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
@@ -57,13 +56,20 @@ func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (str
 	return result.Token, nil
 }
 
-func (c *Client) AuthorizeOCIRequest(ctx context.Context, image oci.Reference, req *http.Request) error {
-	token, err := c.GetRegistryToken(ctx, image)
+func (c *Client) HandleAuth(r *http.Request) error {
+	name := oci.NameFromAPI(r.URL.Path)
+	if (r.Host != "docker.io" && !strings.HasSuffix(r.Host, ".docker.io")) || name == "" {
+		return nil
+	}
+
+	token, err := c.GetRegistryToken(r.Context(), name)
 	if err != nil {
 		return err
 	}
 
-	return oci.AuthorizerToken(token).AuthorizeOCIRequest(ctx, image, req)
+	r.Header.Set("Authorization", "Bearer "+token)
+
+	return nil
 }
 
 func (c *Client) GetRepository(ctx context.Context, image oci.Reference) (*Repository, error) {

internal/dockerhub/client_test.go

@@ -30,8 +30,8 @@ func TestClientGetManifest(t *testing.T) {
 	require.NoError(t, err)
 
 	ociClient := &oci.Client{
-		Client:     client.Client,
-		Authorizer: client,
+		Client:   client.Client,
+		AuthFunc: client.HandleAuth,
 	}
 
 	actual, err := ociClient.GetManifests(context.TODO(), ref)
@@ -53,8 +53,8 @@ func TestClientGetAnnotations(t *testing.T) {
 	require.NoError(t, err)
 
 	ociClient := &oci.Client{
-		Client:     client.Client,
-		Authorizer: client,
+		Client:   client.Client,
+		AuthFunc: client.HandleAuth,
 	}
 
 	manifests, err := ociClient.GetAnnotations(context.TODO(), ref, nil)
@@ -105,7 +105,10 @@ func TestGetTags(t *testing.T) {
 		Client: httputil.NewClient(cachetest.NewCache(t), 24*time.Hour),
 	}
 
-	ociClient := oci.Client{Client: client.Client, Authorizer: client}
+	ociClient := oci.Client{
+		Client:   client.Client,
+		AuthFunc: client.HandleAuth,
+	}
 
 	ref, err := oci.ParseReference("mongo")
 	require.NoError(t, err)

internal/ghcr/client.go

@@ -11,13 +11,11 @@ import (
 	"github.com/AlexGustafsson/cupdate/internal/oci"
 )
 
-var _ oci.Authorizer = (*Client)(nil)
-
 type Client struct {
 	Client *httputil.Client
 }
 
-func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (string, error) {
+func (c *Client) GetRegistryToken(ctx context.Context, repository string) (string, error) {
 	// TODO: Registries expose the realm and scheme via Www-Authenticate if 403
 	// is given
 	u, err := url.Parse("https://ghcr.io/token?service=ghcr.io")
@@ -26,7 +24,7 @@ func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (str
 	}
 
 	query := u.Query()
-	query.Set("scope", fmt.Sprintf("repository:%s:pull", image.Path))
+	query.Set("scope", fmt.Sprintf("repository:%s:pull", repository))
 	u.RawQuery = query.Encode()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
@@ -53,11 +51,18 @@ func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (str
 	return result.Token, nil
 }
 
-func (c *Client) AuthorizeOCIRequest(ctx context.Context, image oci.Reference, req *http.Request) error {
-	token, err := c.GetRegistryToken(ctx, image)
+func (c *Client) HandleAuth(r *http.Request) error {
+	name := oci.NameFromAPI(r.URL.Path)
+	if r.Host != "ghcr.io" || name == "" {
+		return nil
+	}
+
+	token, err := c.GetRegistryToken(r.Context(), name)
 	if err != nil {
 		return err
 	}
 
-	return oci.AuthorizerToken(token).AuthorizeOCIRequest(ctx, image, req)
+	r.Header.Set("Authorization", "Bearer "+token)
+
+	return nil
 }

internal/ghcr/client_test.go

@@ -28,8 +28,8 @@ func TestClientGetManifest(t *testing.T) {
 	require.NoError(t, err)
 
 	ociClient := &oci.Client{
-		Client:     client.Client,
-		Authorizer: client,
+		Client:   client.Client,
+		AuthFunc: client.HandleAuth,
 	}
 
 	actual, err := ociClient.GetManifests(context.TODO(), ref)
@@ -51,8 +51,8 @@ func TestClientGetAnnotations(t *testing.T) {
 	require.NoError(t, err)
 
 	ociClient := &oci.Client{
-		Client:     client.Client,
-		Authorizer: client,
+		Client:   client.Client,
+		AuthFunc: client.HandleAuth,
 	}
 
 	manifests, err := ociClient.GetAnnotations(context.TODO(), ref, nil)

internal/github/client_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/AlexGustafsson/cupdate/internal/cachetest"
 	"github.com/AlexGustafsson/cupdate/internal/httputil"
 	"github.com/AlexGustafsson/cupdate/internal/oci"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 

internal/gitlab/client.go

@@ -17,8 +17,6 @@ import (
 
 var readmePathRegexp = regexp.MustCompile(`href="(.*?/blob/.*?)"`)
 
-var _ oci.Authorizer = (*Client)(nil)
-
 type Client struct {
 	Client *httputil.Client
 }
@@ -168,7 +166,7 @@ func (c *Client) GetBlob(ctx context.Context, href string, includeRaw bool) (*Bl
 	return &blob, nil
 }
 
-func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (string, error) {
+func (c *Client) GetRegistryToken(ctx context.Context, repository string) (string, error) {
 	// TODO: Registries expose the realm and scheme via Www-Authenticate if 403
 	// is given
 	u, err := url.Parse("https://gitlab.com/jwt/auth?service=container_registry")
@@ -177,7 +175,7 @@ func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (str
 	}
 
 	query := u.Query()
-	query.Set("scope", fmt.Sprintf("repository:%s:pull", image.Path))
+	query.Set("scope", fmt.Sprintf("repository:%s:pull", repository))
 	u.RawQuery = query.Encode()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
@@ -207,13 +205,20 @@ func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (str
 	return result.Token, nil
 }
 
-func (c *Client) AuthorizeOCIRequest(ctx context.Context, image oci.Reference, req *http.Request) error {
-	token, err := c.GetRegistryToken(ctx, image)
+func (c *Client) HandleAuth(r *http.Request) error {
+	name := oci.NameFromAPI(r.URL.Path)
+	if r.Host != "registry.gitlab.com" || name == "" {
+		return nil
+	}
+
+	token, err := c.GetRegistryToken(r.Context(), name)
 	if err != nil {
 		return err
 	}
 
-	return oci.AuthorizerToken(token).AuthorizeOCIRequest(ctx, image, req)
+	r.Header.Set("Authorization", "Bearer "+token)
+
+	return nil
 }
 
 func (c *Client) GetProjectContainerRepositories(ctx context.Context, fullPath string) ([]ContainerRepository, error) {