diff --git a/staging/src/k8s.io/apiserver/pkg/admission/config.go b/staging/src/k8s.io/apiserver/pkg/admission/config.go
index e716e62238abe..07c2c90f83f4e 100644
--- a/staging/src/k8s.io/apiserver/pkg/admission/config.go
+++ b/staging/src/k8s.io/apiserver/pkg/admission/config.go
@@ -95,6 +95,18 @@ func ReadAdmissionConfiguration(pluginNames []string, configFilePath string, con
 	if !(runtime.IsMissingVersion(err) || runtime.IsMissingKind(err) || runtime.IsNotRegisteredError(err)) {
 		return nil, err
 	}
+
+	// Only tolerate load errors if the file appears to be one of the two legacy plugin configs
+	unstructuredData := map[string]interface{}{}
+	if err2 := yaml.Unmarshal(data, &unstructuredData); err2 != nil {
+		return nil, err
+	}
+	_, isLegacyImagePolicy := unstructuredData["imagePolicy"]
+	_, isLegacyPodNodeSelector := unstructuredData["podNodeSelectorPluginConfig"]
+	if !isLegacyImagePolicy && !isLegacyPodNodeSelector {
+		return nil, err
+	}
+
 	// convert the legacy format to the new admission control format
 	// in order to preserve backwards compatibility, we set plugins that
 	// previously read input from a non-versioned file configuration to the
diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/admission.go b/staging/src/k8s.io/apiserver/pkg/server/options/admission.go
index 66b0b97ba6359..13458e5daccff 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/options/admission.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/options/admission.go
@@ -38,11 +38,11 @@ import (
 	"k8s.io/client-go/rest"
 )
 
-var scheme = runtime.NewScheme()
+var configScheme = runtime.NewScheme()
 
 func init() {
-	apiserverapi.AddToScheme(scheme)
-	apiserverapiv1alpha1.AddToScheme(scheme)
+	apiserverapi.AddToScheme(configScheme)
+	apiserverapiv1alpha1.AddToScheme(configScheme)
 }
 
 // AdmissionOptions holds the admission options
@@ -125,7 +125,7 @@ func (a *AdmissionOptions) ApplyTo(
 		pluginNames = a.enabledPluginNames()
 	}
 
-	pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, scheme)
+	pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
 	if err != nil {
 		return fmt.Errorf("failed to read plugin config: %v", err)
 	}