Bindings forwarder implementation (#476)

* chore: Copy in proto code This will likely live in ngrok-go long term, but we need to start consuming it now * feat: Add additional printer columns * feat: Add bindings connection handler to wire it all up
ngrok · Oct 30, 2024 · 867460e · 867460e
1 parent fbf5b20
commit 867460e
Show file tree

Hide file tree

Showing 17 changed files with 577 additions and 62 deletions.
diff --git a/api/ngrok/v1alpha1/kubernetesoperator_types.go b/api/ngrok/v1alpha1/kubernetesoperator_types.go
@@ -96,6 +96,10 @@ type KubernetesOperatorStatus struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:validation:MaxLength=4096
 	RegistrationErrorMessage string `json:"errorMessage,omitempty"`
+
+	// EnabledFeatures is the string representation of the features enabled for this Kubernetes Operator
+	// +kubebuilder:validation:Optional
+	EnabledFeatures string `json:"enabledFeatures,omitempty"`
 }
 
 const (
@@ -138,6 +142,9 @@ type KubernetesOperatorSpec struct {
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="ID",type=string,JSONPath=`.status.id`,description="Kubernetes Operator ID"
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.registrationStatus"
+// +kubebuilder:printcolumn:name="Enabled Features",type="string",JSONPath=".status.enabledFeatures"
+// +kubebuilder:printcolumn:name="Binding Name", type="string", JSONPath=".spec.binding.name",priority=2
+// +kubebuilder:printcolumn:name="Binding Ingress Endpoint", type="string", JSONPath=".spec.binding.ingressEndpoint",priority=2
 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,description="Age"
 
 // KubernetesOperator is the Schema for the ngrok kubernetesoperators API

diff --git a/cmd/api/main.go b/cmd/api/main.go
@@ -111,6 +111,7 @@ type managerOpts struct {
 		name               string
 		serviceAnnotations string
 		serviceLabels      string
+		ingressEndpoint    string
 	}
 
 	// env vars
@@ -153,6 +154,7 @@ func cmd() *cobra.Command {
 	c.Flags().StringVar(&opts.bindings.name, "bindings-name", "default", "Name of the Endpoint Binding Configuration")
 	c.Flags().StringVar(&opts.bindings.serviceAnnotations, "bindings-service-annotations", "", "Service Annotations to propagate to the target service")
 	c.Flags().StringVar(&opts.bindings.serviceLabels, "bindings-service-labels", "", "Service Labels to propagate to the target service")
+	c.Flags().StringVar(&opts.bindings.ingressEndpoint, "bindings-ingress-endpoint", "", "The endpoint the bindings forwarder connects to")
 
 	opts.zapOpts = &zap.Options{}
 	goFlagSet := flag.NewFlagSet("manager", flag.ContinueOnError)
@@ -549,6 +551,9 @@ func createKubernetesOperator(ctx context.Context, client client.Client, opts ma
 				TlsSecretName: "ngrok-operator-default-tls",
 				AllowedURLs:   opts.bindings.allowedURLs,
 			}
+			if opts.bindings.ingressEndpoint != "" {
+				k8sOperator.Spec.Binding.IngressEndpoint = &opts.bindings.ingressEndpoint
+			}
 		}
 		k8sOperator.Spec.EnabledFeatures = features
 		return nil

diff --git a/cmd/bindings-forwarder/main.go b/cmd/bindings-forwarder/main.go
@@ -40,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	bindingsv1alpha1 "github.com/ngrok/ngrok-operator/api/bindings/v1alpha1"
+	ngrokv1alpha1 "github.com/ngrok/ngrok-operator/api/ngrok/v1alpha1"
 	bindingscontroller "github.com/ngrok/ngrok-operator/internal/controller/bindings"
 	"github.com/ngrok/ngrok-operator/internal/version"
 	"github.com/ngrok/ngrok-operator/pkg/bindingsdriver"
@@ -54,6 +55,7 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(bindingsv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(ngrokv1alpha1.AddToScheme(scheme))
 }
 
 func main() {
@@ -65,6 +67,7 @@ func main() {
 
 type managerOpts struct {
 	// flags
+	releaseName string
 	metricsAddr string
 	probeAddr   string
 	description string
@@ -84,6 +87,7 @@ func cmd() *cobra.Command {
 		},
 	}
 
+	c.Flags().StringVar(&opts.releaseName, "release-name", "ngrok-operator", "Helm Release name for the deployed operator")
 	c.Flags().StringVar(&opts.metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to")
 	c.Flags().StringVar(&opts.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	c.Flags().StringVar(&opts.description, "description", "Created by the ngrok-operator", "Description for this installation")
@@ -134,11 +138,12 @@ func runController(_ context.Context, opts managerOpts) error {
 	bd := bindingsdriver.New()
 
 	if err = (&bindingscontroller.ForwarderReconciler{
-		Client:         mgr.GetClient(),
-		Log:            ctrl.Log.WithName("controllers").WithName("bindings-forwarder"),
-		Scheme:         mgr.GetScheme(),
-		Recorder:       mgr.GetEventRecorderFor("bindings-forwarder-controller"),
-		BindingsDriver: bd,
+		Client:                 mgr.GetClient(),
+		Log:                    ctrl.Log.WithName("controllers").WithName("bindings-forwarder"),
+		Scheme:                 mgr.GetScheme(),
+		Recorder:               mgr.GetEventRecorderFor("bindings-forwarder-controller"),
+		BindingsDriver:         bd,
+		KubernetesOperatorName: opts.releaseName,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "BindingsForwarder")
 		os.Exit(1)

diff --git a/go.mod b/go.mod
@@ -16,6 +16,7 @@ require (
 	golang.ngrok.com/ngrok v1.7.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	golang.org/x/sync v0.5.0
+	google.golang.org/protobuf v1.31.0
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v0.29.2
@@ -95,7 +96,6 @@ require (
 	golang.org/x/tools v0.16.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

diff --git a/helm/ngrok-operator/README.md b/helm/ngrok-operator/README.md
@@ -143,15 +143,16 @@ To uninstall the chart:
 
 ### Kubernetes Bindings feature configuration
 
-| Name                                            | Description                                                                             | Value                       |
-| ----------------------------------------------- | --------------------------------------------------------------------------------------- | --------------------------- |
-| `bindings.enabled`                              | Whether to enable the Endpoint Bindings feature                                         | `false`                     |
-| `bindings.name`                                 | Unique name of this kubernetes binding in your ngrok account                            | `""`                        |
-| `bindings.description`                          | Description of this kubernetes binding in your ngrok account                            | `Created by ngrok-operator` |
-| `bindings.allowedURLs`                          | List of allowed endpoint URL formats that this binding will project into the cluster    | `["*"]`                     |
-| `bindings.serviceAnnotations`                   | Annotations to add to projected services bound to an endpoint                           | `{}`                        |
-| `bindings.serviceLabels`                        | Labels to add to projected services bound to an endpoint                                | `{}`                        |
-| `bindings.forwarder.replicaCount`               | The number of bindings forwarders to run.                                               | `1`                         |
-| `bindings.forwarder.serviceAccount.create`      | Specifies whether a ServiceAccount should be created for the bindings forwarder pod(s). | `true`                      |
-| `bindings.forwarder.serviceAccount.name`        | The name of the ServiceAccount to use for the bindings forwarder pod(s).                | `""`                        |
-| `bindings.forwarder.serviceAccount.annotations` | Additional annotations to add to the bindings-forwarder ServiceAccount                  | `{}`                        |
+| Name                                            | Description                                                                             | Value                                     |
+| ----------------------------------------------- | --------------------------------------------------------------------------------------- | ----------------------------------------- |
+| `bindings.enabled`                              | Whether to enable the Endpoint Bindings feature                                         | `false`                                   |
+| `bindings.name`                                 | Unique name of this kubernetes binding in your ngrok account                            | `""`                                      |
+| `bindings.description`                          | Description of this kubernetes binding in your ngrok account                            | `Created by ngrok-operator`               |
+| `bindings.allowedURLs`                          | List of allowed endpoint URL formats that this binding will project into the cluster    | `["*"]`                                   |
+| `bindings.serviceAnnotations`                   | Annotations to add to projected services bound to an endpoint                           | `{}`                                      |
+| `bindings.serviceLabels`                        | Labels to add to projected services bound to an endpoint                                | `{}`                                      |
+| `bindings.ingressEndpoint`                      | The hostname of the ingress endpoint for the bindings                                   | `kubernetes-binding-ingress.ngrok.io:443` |
+| `bindings.forwarder.replicaCount`               | The number of bindings forwarders to run.                                               | `1`                                       |
+| `bindings.forwarder.serviceAccount.create`      | Specifies whether a ServiceAccount should be created for the bindings forwarder pod(s). | `true`                                    |
+| `bindings.forwarder.serviceAccount.name`        | The name of the ServiceAccount to use for the bindings forwarder pod(s).                | `""`                                      |
+| `bindings.forwarder.serviceAccount.annotations` | Additional annotations to add to the bindings-forwarder ServiceAccount                  | `{}`                                      |
diff --git a/helm/ngrok-operator/templates/bindings-forwarder/deployment.yaml b/helm/ngrok-operator/templates/bindings-forwarder/deployment.yaml
@@ -62,6 +62,7 @@ spec:
         command:
         - /bindings-forwarder-manager
         args:
+        - --release-name={{ .Release.Name }}
         {{- if .Values.description }}
         - --description={{ .Values.description | quote }}
         {{- end }}

diff --git a/helm/ngrok-operator/templates/bindings-forwarder/rbac.yaml b/helm/ngrok-operator/templates/bindings-forwarder/rbac.yaml
@@ -25,6 +25,29 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ngrok.k8s.ngrok.com
+  resources:
+  - kubernetesoperators
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

diff --git a/helm/ngrok-operator/templates/controller-deployment.yaml b/helm/ngrok-operator/templates/controller-deployment.yaml
@@ -80,6 +80,7 @@ spec:
           {{- end }}
           {{- $serviceLabels | join "," }}
         {{- end }}
+        - --bindings-ingress-endpoint={{ .Values.bindings.ingressEndpoint }}
         {{- end }}
         {{- if .Values.description }}
         - --description={{ .Values.description | quote }}

diff --git a/helm/ngrok-operator/templates/crds/ngrok.k8s.ngrok.com_kubernetesoperators.yaml b/helm/ngrok-operator/templates/crds/ngrok.k8s.ngrok.com_kubernetesoperators.yaml
diff --git a/helm/ngrok-operator/tests/bindings-forwarder/__snapshot__/deployment_test.yaml.snap b/helm/ngrok-operator/tests/bindings-forwarder/__snapshot__/deployment_test.yaml.snap
diff --git a/helm/ngrok-operator/tests/bindings-forwarder/__snapshot__/rbac_test.yaml.snap b/helm/ngrok-operator/tests/bindings-forwarder/__snapshot__/rbac_test.yaml.snap
diff --git a/helm/ngrok-operator/values.schema.json b/helm/ngrok-operator/values.schema.json
diff --git a/helm/ngrok-operator/values.yaml b/helm/ngrok-operator/values.yaml
@@ -274,6 +274,7 @@ gateway:
 ## @param bindings.allowedURLs List of allowed endpoint URL formats that this binding will project into the cluster
 ## @param bindings.serviceAnnotations Annotations to add to projected services bound to an endpoint
 ## @param bindings.serviceLabels Labels to add to projected services bound to an endpoint
+## @param bindings.ingressEndpoint The hostname of the ingress endpoint for the bindings
 ##
 bindings:
   enabled: false # in-development
@@ -283,6 +284,7 @@ bindings:
   - "*" # TODO(hkatz) only supporting this for now, implement others later
   serviceAnnotations: {}
   serviceLabels: {}
+  ingressEndpoint: "kubernetes-binding-ingress.ngrok.io:443"
 
   forwarder:
     ## @param bindings.forwarder.replicaCount The number of bindings forwarders to run.