Merge pull request kubernetes#49495 from deads2k/controller-12-tolera…

…tion

Automatic merge from submit-queue (batch tested with PRs 49665, 49689, 49495, 49146, 48934)

make it possible to allow discovery errors for controllers

Update the discovery client to return partial discovery information *and* an error.  Since we can aggregate API servers, discovery of some resources can fail independently.  Callers of this function who want to tolerate the errors can, existing callers will still get an error and fail in normal blocks.

@kubernetes/sig-api-machinery-misc @sttts

cmd/kube-controller-manager/app/BUILD

@@ -111,6 +111,7 @@ go_library(
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",

cmd/kube-controller-manager/app/controllermanager.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 
@@ -366,7 +367,10 @@ func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (ma
 
 	resourceMap, err := discoveryClient.ServerResources()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get supported resources from server: %v", err)
+		utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
+	}
+	if len(resourceMap) == 0 {
+		return nil, fmt.Errorf("unable to get any supported resources from server")
 	}
 
 	allResources := map[schema.GroupVersionResource]bool{}

cmd/kube-controller-manager/app/core.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
@@ -302,7 +303,10 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
 	gcClientset := ctx.ClientBuilder.ClientOrDie("generic-garbage-collector")
 	preferredResources, err := gcClientset.Discovery().ServerPreferredResources()
 	if err != nil {
-		return true, fmt.Errorf("failed to get supported resources from server: %v", err)
+		utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
+	}
+	if len(preferredResources) == 0 {
+		return true, fmt.Errorf("unable to get any supported resources from server: %v", err)
 	}
 	deletableResources := discovery.FilteredBy(discovery.SupportsAllVerbs{Verbs: []string{"get", "list", "watch", "patch", "update", "delete"}}, preferredResources)
 	deletableGroupVersionResources, err := discovery.GroupVersionResources(deletableResources)

pkg/controller/namespace/deletion/BUILD

@@ -20,6 +20,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",

pkg/controller/namespace/deletion/namespaced_resources_deleter.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
@@ -165,7 +166,10 @@ func (d *namespacedResourcesDeleter) initOpCache() {
 	// TODO(sttts): get rid of opCache and http 405 logic around it and trust discovery info
 	resources, err := d.discoverResourcesFn()
 	if err != nil {
-		glog.Fatalf("Failed to get supported resources: %v", err)
+		utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
+	}
+	if len(resources) == 0 {
+		glog.Fatalf("Unable to get any supported resources from server: %v", err)
 	}
 	deletableGroupVersionResources := []schema.GroupVersionResource{}
 	for _, rl := range resources {

staging/src/k8s.io/client-go/discovery/discovery_client.go

@@ -187,7 +187,7 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r
 }
 
 // serverResources returns the supported resources for all groups and versions.
-func (d *DiscoveryClient) serverResources(failEarly bool) ([]*metav1.APIResourceList, error) {
+func (d *DiscoveryClient) serverResources() ([]*metav1.APIResourceList, error) {
 	apiGroups, err := d.ServerGroups()
 	if err != nil {
 		return nil, err
@@ -203,9 +203,6 @@ func (d *DiscoveryClient) serverResources(failEarly bool) ([]*metav1.APIResource
 			if err != nil {
 				// TODO: maybe restrict this to NotFound errors
 				failedGroups[gv] = err
-				if failEarly {
-					return nil, &ErrGroupDiscoveryFailed{Groups: failedGroups}
-				}
 				continue
 			}
 
@@ -249,7 +246,7 @@ func IsGroupDiscoveryFailedError(err error) bool {
 }
 
 // serverPreferredResources returns the supported resources with the version preferred by the server.
-func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.APIResourceList, error) {
+func (d *DiscoveryClient) serverPreferredResources() ([]*metav1.APIResourceList, error) {
 	serverGroupList, err := d.ServerGroups()
 	if err != nil {
 		return nil, err
@@ -269,9 +266,6 @@ func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.AP
 			if err != nil {
 				// TODO: maybe restrict this to NotFound errors
 				failedGroups[groupVersion] = err
-				if failEarly {
-					return nil, &ErrGroupDiscoveryFailed{Groups: failedGroups}
-				}
 				continue
 			}
 
@@ -316,9 +310,7 @@ func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.AP
 // ServerPreferredResources returns the supported resources with the version preferred by the
 // server.
 func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
-	return withRetries(defaultRetries, func(retryEarly bool) ([]*metav1.APIResourceList, error) {
-		return d.serverPreferredResources(retryEarly)
-	})
+	return withRetries(defaultRetries, d.serverPreferredResources)
 }
 
 // ServerPreferredNamespacedResources returns the supported namespaced resources with the
@@ -394,12 +386,11 @@ func (d *DiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) {
 }
 
 // withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns.
-func withRetries(maxRetries int, f func(failEarly bool) ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) {
+func withRetries(maxRetries int, f func() ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) {
 	var result []*metav1.APIResourceList
 	var err error
 	for i := 0; i < maxRetries; i++ {
-		failEarly := i < maxRetries-1
-		result, err = f(failEarly)
+		result, err = f()
 		if err == nil {
 			return result, nil
 		}