From a2987a21837bbcecd55fa92c65f9cd5a28557851 Mon Sep 17 00:00:00 2001
From: Debarati Basu-Nag <dbasunag@redhat.com>
Date: Thu, 12 Jan 2023 10:49:19 -0500
Subject: [PATCH] add ForbiddenError to the retry list and add default
 exceptions to a few calls (#1058)

* add ForbiddenError to the retry list

* add default exceptions to a few calls
---
 ocp_resources/constants.py       | 22 ++++++++++++++++++++--
 ocp_resources/resource.py        | 28 +++++++---------------------
 ocp_resources/virtual_machine.py | 11 +++++++++--
 3 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/ocp_resources/constants.py b/ocp_resources/constants.py
index 9b317fe836..713d8c532e 100644
--- a/ocp_resources/constants.py
+++ b/ocp_resources/constants.py
@@ -1,7 +1,25 @@
-from openshift.dynamic.exceptions import NotFoundError
-from urllib3.exceptions import ProtocolError
+from kubernetes.dynamic.exceptions import ForbiddenError
+from openshift.dynamic.exceptions import (
+    InternalServerError,
+    NotFoundError,
+    ServerTimeoutError,
+)
+from urllib3.exceptions import MaxRetryError, ProtocolError
 
 
+DEFAULT_CLUSTER_RETRY_EXCEPTIONS = {
+    MaxRetryError: [],
+    ConnectionAbortedError: [],
+    ConnectionResetError: [],
+    InternalServerError: [
+        "etcdserver: leader changed",
+        "etcdserver: request timed out",
+        "Internal error occurred: failed calling webhook",
+        "rpc error:",
+    ],
+    ServerTimeoutError: [],
+    ForbiddenError: ["context deadline exceeded"],
+}
 PROTOCOL_ERROR_EXCEPTION_DICT = {ProtocolError: []}
 NOT_FOUND_ERROR_EXCEPTION_DICT = {NotFoundError: []}
 TIMEOUT_1MINUTE = 60
diff --git a/ocp_resources/resource.py b/ocp_resources/resource.py
index bee8323c38..da2c1e6759 100644
--- a/ocp_resources/resource.py
+++ b/ocp_resources/resource.py
@@ -10,17 +10,12 @@
 import yaml
 from kubernetes.dynamic.exceptions import ForbiddenError, MethodNotAllowedError
 from openshift.dynamic import DynamicClient
-from openshift.dynamic.exceptions import (
-    ConflictError,
-    InternalServerError,
-    NotFoundError,
-    ServerTimeoutError,
-)
+from openshift.dynamic.exceptions import ConflictError, NotFoundError
 from openshift.dynamic.resource import ResourceField
 from packaging.version import Version
-from urllib3.exceptions import MaxRetryError
 
 from ocp_resources.constants import (
+    DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
     NOT_FOUND_ERROR_EXCEPTION_DICT,
     PROTOCOL_ERROR_EXCEPTION_DICT,
     TIMEOUT_1MINUTE,
@@ -35,19 +30,6 @@
 )
 
 
-DEFAULT_CLUSTER_RETRY_EXCEPTIONS = {
-    MaxRetryError: [],
-    ConnectionAbortedError: [],
-    ConnectionResetError: [],
-    InternalServerError: [
-        "etcdserver: leader changed",
-        "etcdserver: request timed out",
-        "Internal error occurred: failed calling webhook",
-        "rpc error:",
-    ],
-    ServerTimeoutError: [],
-}
-
 LOGGER = get_logger(__name__)
 MAX_SUPPORTED_API_VERSION = "v2"
 
@@ -586,6 +568,7 @@ def wait(self, timeout=TIMEOUT_4MINUTES, sleep=1):
             exceptions_dict={
                 **PROTOCOL_ERROR_EXCEPTION_DICT,
                 **NOT_FOUND_ERROR_EXCEPTION_DICT,
+                **DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
             },
             func=lambda: self.exists,
         )
@@ -652,7 +635,10 @@ def wait_for_status(
         samples = TimeoutSampler(
             wait_timeout=timeout,
             sleep=sleep,
-            exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT,
+            exceptions_dict={
+                **PROTOCOL_ERROR_EXCEPTION_DICT,
+                **DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
+            },
             func=self.api.get,
             field_selector=f"metadata.name=={self.name}",
             namespace=self.namespace,
diff --git a/ocp_resources/virtual_machine.py b/ocp_resources/virtual_machine.py
index 8f06759a4d..a7488344cf 100644
--- a/ocp_resources/virtual_machine.py
+++ b/ocp_resources/virtual_machine.py
@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 
 
-from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES
+from ocp_resources.constants import (
+    DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
+    PROTOCOL_ERROR_EXCEPTION_DICT,
+    TIMEOUT_4MINUTES,
+)
 from ocp_resources.resource import NamespacedResource
 from ocp_resources.utils import TimeoutSampler
 from ocp_resources.virtual_machine_instance import VirtualMachineInstance
@@ -109,7 +113,10 @@ def wait_for_ready_status(self, status, timeout=TIMEOUT_4MINUTES, sleep=1):
         samples = TimeoutSampler(
             wait_timeout=timeout,
             sleep=sleep,
-            exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT,
+            exceptions_dict={
+                **PROTOCOL_ERROR_EXCEPTION_DICT,
+                **DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
+            },
             func=self.api.get,
             field_selector=f"metadata.name=={self.name}",
             namespace=self.namespace,