[ws-manager-mk2] allow the k8s client a higher limit (gitpod-io#19695)

This helps unblock consumers, such as gRPC and controller reconcilation.

The side effects are:
* the longest running reconcilation for mk2 is no longer 2s, it floats between 250-600ms.
* mk2  workqueue latency is consistent, and the duration doesn't become excessive anymore.
* mk2 work queue depth is high now, but, flattens as start workspace requests cease, and drops as workspaces stop (MarkActive volume)
* increased memory utilization for the API server (it doubles)

components/ws-manager-mk2/main.go

@@ -18,6 +18,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	"github.com/bombsimon/logrusr/v4"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -140,6 +141,17 @@ func main() {
 		LeaderElection:                true,
 		LeaderElectionID:              "ws-manager-mk2-leader.gitpod.io",
 		LeaderElectionReleaseOnCancel: true,
+		NewClient: func(config *rest.Config, options client.Options) (client.Client, error) {
+			config.QPS = 100
+			config.Burst = 150
+
+			c, err := client.New(config, options)
+			if err != nil {
+				return nil, err
+			}
+
+			return c, nil
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")