From aa28bc706727f0cb855f02e1112cf5a4b50b32af Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Tue, 26 Mar 2019 19:11:57 +0100 Subject: [PATCH 01/16] Allow create from string or from dict This is a fix for #722 --- kubernetes/utils/__init__.py | 3 +- kubernetes/utils/create_from_yaml.py | 71 ++++++++++++++++------------ 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/kubernetes/utils/__init__.py b/kubernetes/utils/__init__.py index 2b8597c2fb..1f84f3898f 100644 --- a/kubernetes/utils/__init__.py +++ b/kubernetes/utils/__init__.py @@ -14,4 +14,5 @@ from __future__ import absolute_import -from .create_from_yaml import FailToCreateError, create_from_yaml +from .create_from_yaml import (FailToCreateError, create_from_yaml, + create_from_map) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 29be813b1f..32ae24160c 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - +import io import re + from os import path import yaml @@ -30,7 +30,7 @@ def create_from_yaml( Perform an action from a yaml file. Pass True for verbose to print confirmation information. Input: - yaml_file: string. Contains the path to yaml file. + yaml_file: string. Contains yaml string or a path to yaml file. k8s_client: an ApiClient object, initialized with the client args. verbose: If True, print confirmation from the create action. Default is False. @@ -54,35 +54,44 @@ def create_from_yaml( processing of the request. Valid values are: - All: all dry run stages will be processed """ + if path.exists(yaml_file): + with open(path.abspath(yaml_file)) as f: + yaml_file = io.StringIO(f.read()) + + yml_document_all = yaml.safe_load_all(yaml_file) + # Load all documents from a single YAML file + for yml_document in yml_document_all: + create_from_map(k8s_client, yml_document, verbose, + **kwargs) + + +def create_from_map(k8s_client, yml_document, verbose=False, **kwargs): + # If it is a list type, will need to iterate its items + api_exceptions = [] + + if "List" in yml_document["kind"]: + # Could be "List" or "Pod/Service/...List" + # This is a list type. iterate within its items + kind = yml_document["kind"].replace("List", "") + for yml_object in yml_document["items"]: + # Mitigate cases when server returns a xxxList object + # See kubernetes-client/python#586 + if kind is not "": + yml_object["apiVersion"] = yml_document["apiVersion"] + yml_object["kind"] = kind + try: + create_from_yaml_single_item( + k8s_client, yml_object, verbose, **kwargs) + except client.rest.ApiException as api_exception: + api_exceptions.append(api_exception) + else: + # This is a single object. Call the single item method + try: + create_from_yaml_single_item( + k8s_client, yml_document, verbose, **kwargs) + except client.rest.ApiException as api_exception: + api_exceptions.append(api_exception) - with open(path.abspath(yaml_file)) as f: - yml_document_all = yaml.safe_load_all(f) - api_exceptions = [] - # Load all documents from a single YAML file - for yml_document in yml_document_all: - # If it is a list type, will need to iterate its items - if "List" in yml_document["kind"]: - # Could be "List" or "Pod/Service/...List" - # This is a list type. iterate within its items - kind = yml_document["kind"].replace("List", "") - for yml_object in yml_document["items"]: - # Mitigate cases when server returns a xxxList object - # See kubernetes-client/python#586 - if kind is not "": - yml_object["apiVersion"] = yml_document["apiVersion"] - yml_object["kind"] = kind - try: - create_from_yaml_single_item( - k8s_client, yml_object, verbose, **kwargs) - except client.rest.ApiException as api_exception: - api_exceptions.append(api_exception) - else: - # This is a single object. Call the single item method - try: - create_from_yaml_single_item( - k8s_client, yml_document, verbose, **kwargs) - except client.rest.ApiException as api_exception: - api_exceptions.append(api_exception) # In case we have exceptions waiting for us, raise them if api_exceptions: raise FailToCreateError(api_exceptions) From cee4e49edad0f7c3a8081331c5f6a0e6afee731b Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Tue, 26 Mar 2019 23:30:25 +0100 Subject: [PATCH 02/16] Rename create_from_map to create_from_dict, add tests --- kubernetes/e2e_test/test_utils.py | 34 ++++++++++++++++++++++++++++ kubernetes/utils/create_from_yaml.py | 6 ++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/kubernetes/e2e_test/test_utils.py b/kubernetes/e2e_test/test_utils.py index 6c40361ff8..bf598dca7f 100644 --- a/kubernetes/e2e_test/test_utils.py +++ b/kubernetes/e2e_test/test_utils.py @@ -14,6 +14,7 @@ import unittest +import yaml from kubernetes import utils, client from kubernetes.e2e_test import base @@ -42,6 +43,39 @@ def test_create_apps_deployment_from_yaml(self): name="nginx-app", namespace="default", body={}) + def test_create_apps_deployment_from_yaml_string(self): + k8s_client = client.api_client.ApiClient(configuration=self.config) + with open(self.path_prefix + "apps-deployment.yaml") as f: + yaml_str = f.read() + + utils.create_from_yaml( + k8s_client, yaml_str) + + app_api = client.AppsV1beta1Api(k8s_client) + dep = app_api.read_namespaced_deployment(name="nginx-app", + namespace="default") + self.assertIsNotNone(dep) + app_api.delete_namespaced_deployment( + name="nginx-app", namespace="default", + body={}) + + def test_create_apps_deployment_from_yaml_obj(self): + k8s_client = client.api_client.ApiClient(configuration=self.config) + with open(self.path_prefix + "apps-deployment.yaml") as f: + yml_obj = yaml.safe_load(f) + + utils.create_from_dict( + k8s_client, yml_obj) + + app_api = client.AppsV1beta1Api(k8s_client) + dep = app_api.read_namespaced_deployment(name="nginx-app", + namespace="default") + self.assertIsNotNone(dep) + app_api.delete_namespaced_deployment( + name="nginx-app", namespace="default", + body={}) + + def test_create_extensions_deployment_from_yaml(self): """ Should be able to create an extensions/v1beta1 deployment. diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 32ae24160c..be5de7862c 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -30,7 +30,7 @@ def create_from_yaml( Perform an action from a yaml file. Pass True for verbose to print confirmation information. Input: - yaml_file: string. Contains yaml string or a path to yaml file. + yaml_file: string. Contains yaml string or a path to yaml file. k8s_client: an ApiClient object, initialized with the client args. verbose: If True, print confirmation from the create action. Default is False. @@ -61,11 +61,11 @@ def create_from_yaml( yml_document_all = yaml.safe_load_all(yaml_file) # Load all documents from a single YAML file for yml_document in yml_document_all: - create_from_map(k8s_client, yml_document, verbose, + create_from_dict(k8s_client, yml_document, verbose, **kwargs) -def create_from_map(k8s_client, yml_document, verbose=False, **kwargs): +def create_from_dict(k8s_client, yml_document, verbose=False, **kwargs): # If it is a list type, will need to iterate its items api_exceptions = [] From 18f45c9ceca522d1202cb2b5b7c98f32321fe516 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Fri, 5 Apr 2019 06:30:43 +0200 Subject: [PATCH 03/16] Fix wrong import name --- kubernetes/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/utils/__init__.py b/kubernetes/utils/__init__.py index 1f84f3898f..91fb586332 100644 --- a/kubernetes/utils/__init__.py +++ b/kubernetes/utils/__init__.py @@ -15,4 +15,4 @@ from __future__ import absolute_import from .create_from_yaml import (FailToCreateError, create_from_yaml, - create_from_map) + create_from_dict) From 0779fc9c66016d3544aab5e35457f86b25fcd961 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Fri, 5 Apr 2019 08:20:57 +0200 Subject: [PATCH 04/16] Fix collection of execptions when applying yaml This fixes of all tests and keeps the original API. It's the users responsiblility to do something the execptions when using `create_from_dict`, no air bag or breaks are supplied here. --- kubernetes/e2e_test/test_utils.py | 1 - kubernetes/utils/create_from_yaml.py | 13 ++++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/kubernetes/e2e_test/test_utils.py b/kubernetes/e2e_test/test_utils.py index dc2be10974..df387a28ab 100644 --- a/kubernetes/e2e_test/test_utils.py +++ b/kubernetes/e2e_test/test_utils.py @@ -75,7 +75,6 @@ def test_create_apps_deployment_from_yaml_obj(self): name="nginx-app", namespace="default", body={}) - def test_create_extensions_deployment_from_yaml(self): """ Should be able to create an extensions/v1beta1 deployment. diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 8b3c19ba1f..cf157da583 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -60,9 +60,16 @@ def create_from_yaml( yml_document_all = yaml.safe_load_all(yaml_file) # Load all documents from a single YAML file + fail_exceptions = [] + for yml_document in yml_document_all: - create_from_dict(k8s_client, yml_document, verbose, - **kwargs) + exceptions = create_from_dict(k8s_client, yml_document, verbose, + **kwargs) + if exceptions: + fail_exceptions.extend(exceptions) + + if fail_exceptions: + raise FailToCreateError(fail_exceptions) def create_from_dict(k8s_client, yml_document, verbose=False, **kwargs): @@ -94,7 +101,7 @@ def create_from_dict(k8s_client, yml_document, verbose=False, **kwargs): # In case we have exceptions waiting for us, raise them if api_exceptions: - raise FailToCreateError(api_exceptions) + return api_exceptions def create_from_yaml_single_item( From 2b83c683f8523a527b1815f0896d47c599aa9b09 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Mon, 8 Apr 2019 12:35:41 +0200 Subject: [PATCH 05/16] Fix for a flaky test failing because k8s is slow We create a deployment and do the following: ``` self.assertIsNotNone(dep) ``` Which does not fail, and then the code proceeds to deletion and fails with a 404 execption in 80% of the time, but sometimes it works. The deployment is there, but for some reason not available for deletion. Travis CI also showed inconsitent behaviour on this. Python3.5 passed but all other version failed. With this commit we wait for the deployment to become available for deletion and only then continue. --- kubernetes/e2e_test/test_utils.py | 34 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/kubernetes/e2e_test/test_utils.py b/kubernetes/e2e_test/test_utils.py index df387a28ab..04eb1f4865 100644 --- a/kubernetes/e2e_test/test_utils.py +++ b/kubernetes/e2e_test/test_utils.py @@ -16,6 +16,7 @@ import yaml from kubernetes import utils, client +from kubernetes.client.rest import ApiException from kubernetes.e2e_test import base @@ -39,9 +40,14 @@ def test_create_apps_deployment_from_yaml(self): dep = app_api.read_namespaced_deployment(name="nginx-app", namespace="default") self.assertIsNotNone(dep) - app_api.delete_namespaced_deployment( - name="nginx-app", namespace="default", - body={}) + while True: + try: + app_api.delete_namespaced_deployment( + name="nginx-app", namespace="default", + body={}) + break + except ApiException: + continue def test_create_apps_deployment_from_yaml_string(self): k8s_client = client.api_client.ApiClient(configuration=self.config) @@ -55,9 +61,14 @@ def test_create_apps_deployment_from_yaml_string(self): dep = app_api.read_namespaced_deployment(name="nginx-app", namespace="default") self.assertIsNotNone(dep) - app_api.delete_namespaced_deployment( - name="nginx-app", namespace="default", - body={}) + while True: + try: + app_api.delete_namespaced_deployment( + name="nginx-app", namespace="default", + body={}) + break + except ApiException: + continue def test_create_apps_deployment_from_yaml_obj(self): k8s_client = client.api_client.ApiClient(configuration=self.config) @@ -71,9 +82,14 @@ def test_create_apps_deployment_from_yaml_obj(self): dep = app_api.read_namespaced_deployment(name="nginx-app", namespace="default") self.assertIsNotNone(dep) - app_api.delete_namespaced_deployment( - name="nginx-app", namespace="default", - body={}) + while True: + try: + app_api.delete_namespaced_deployment( + name="nginx-app", namespace="default", + body={}) + break + except ApiException: + continue def test_create_extensions_deployment_from_yaml(self): """ From 8d41478ed87145af480ccd0cc5863d3ed8ce2ca8 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Mon, 8 Apr 2019 13:54:26 +0200 Subject: [PATCH 06/16] Explicitly use other deployment names for test Using `nginx-app` deployment multiple times, is problematic because we get conflicts or not found error. Instead of trying to handle all cases, explicit different names are used now. The tests now runs more reliably --- kubernetes/e2e_test/test_utils.py | 24 ++++++++----------- .../e2e_test/test_yaml/apps-deployment-2.yaml | 21 ++++++++++++++++ 2 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 kubernetes/e2e_test/test_yaml/apps-deployment-2.yaml diff --git a/kubernetes/e2e_test/test_utils.py b/kubernetes/e2e_test/test_utils.py index 04eb1f4865..58bb57bb5f 100644 --- a/kubernetes/e2e_test/test_utils.py +++ b/kubernetes/e2e_test/test_utils.py @@ -51,20 +51,20 @@ def test_create_apps_deployment_from_yaml(self): def test_create_apps_deployment_from_yaml_string(self): k8s_client = client.api_client.ApiClient(configuration=self.config) - with open(self.path_prefix + "apps-deployment.yaml") as f: + with open(self.path_prefix + "apps-deployment-2.yaml") as f: yaml_str = f.read() utils.create_from_yaml( k8s_client, yaml_str) app_api = client.AppsV1beta1Api(k8s_client) - dep = app_api.read_namespaced_deployment(name="nginx-app", + dep = app_api.read_namespaced_deployment(name="nginx-app-2", namespace="default") self.assertIsNotNone(dep) while True: try: app_api.delete_namespaced_deployment( - name="nginx-app", namespace="default", + name="nginx-app-2", namespace="default", body={}) break except ApiException: @@ -75,21 +75,17 @@ def test_create_apps_deployment_from_yaml_obj(self): with open(self.path_prefix + "apps-deployment.yaml") as f: yml_obj = yaml.safe_load(f) - utils.create_from_dict( - k8s_client, yml_obj) + yml_obj["metadata"]["name"] = "nginx-app-3" + + utils.create_from_dict(k8s_client, yml_obj) app_api = client.AppsV1beta1Api(k8s_client) - dep = app_api.read_namespaced_deployment(name="nginx-app", + dep = app_api.read_namespaced_deployment(name="nginx-app-3", namespace="default") self.assertIsNotNone(dep) - while True: - try: - app_api.delete_namespaced_deployment( - name="nginx-app", namespace="default", - body={}) - break - except ApiException: - continue + app_api.delete_namespaced_deployment( + name="nginx-app-3", namespace="default", + body={}) def test_create_extensions_deployment_from_yaml(self): """ diff --git a/kubernetes/e2e_test/test_yaml/apps-deployment-2.yaml b/kubernetes/e2e_test/test_yaml/apps-deployment-2.yaml new file mode 100644 index 0000000000..b2acb92f82 --- /dev/null +++ b/kubernetes/e2e_test/test_yaml/apps-deployment-2.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: nginx-app-2 + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.15.4 + ports: + - containerPort: 80 From 408d405704d8826dd164d828670fa37bf6ca8027 Mon Sep 17 00:00:00 2001 From: Oz Tiram Date: Tue, 9 Apr 2019 22:51:54 +0200 Subject: [PATCH 07/16] Handle StringIO for Python2 properly --- kubernetes/utils/create_from_yaml.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index cf157da583..9cfec1ab63 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -11,8 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import io import re +import sys + +if sys.version_info.major > 2: + from StringIO import StringIO +else: + from io import StringIO from os import path @@ -56,7 +61,11 @@ def create_from_yaml( """ if path.exists(yaml_file): with open(path.abspath(yaml_file)) as f: - yaml_file = io.StringIO(f.read()) + content = f.read() + try: + yaml_file = StringIO(content) + except TypeError: + yaml_file = StringIO(content.decode('utf-8')) yml_document_all = yaml.safe_load_all(yaml_file) # Load all documents from a single YAML file From 38894cc3970cf5578abd8a94a384a7602a965438 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 9 Apr 2019 23:03:02 +0200 Subject: [PATCH 08/16] Fix import for StringIO This is embarrassing. --- kubernetes/utils/create_from_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 9cfec1ab63..46ee53c3cf 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -14,7 +14,7 @@ import re import sys -if sys.version_info.major > 2: +if sys.version_info.major < 3: from StringIO import StringIO else: from io import StringIO From 139848efcb7a78e8577ac6c9a5b49abe97340e21 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 9 Apr 2019 23:24:16 +0200 Subject: [PATCH 09/16] Relax pycodestyle: import only allowed at the top While this is a really good convention, sometimes one must import stuff inside a try except block. This block is still at the top, but pycodestyle treats like it isn't, because it's in an idented block, and the outeer scope. --- kubernetes/utils/__init__.py | 4 ++-- kubernetes/utils/create_from_yaml.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/kubernetes/utils/__init__.py b/kubernetes/utils/__init__.py index 91fb586332..72f55c7511 100644 --- a/kubernetes/utils/__init__.py +++ b/kubernetes/utils/__init__.py @@ -14,5 +14,5 @@ from __future__ import absolute_import -from .create_from_yaml import (FailToCreateError, create_from_yaml, - create_from_dict) +from .create_from_yaml import (FailToCreateError, create_from_dict, + create_from_yaml) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 46ee53c3cf..aecc3672a6 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -13,18 +13,17 @@ # limitations under the License. import re import sys - -if sys.version_info.major < 3: - from StringIO import StringIO -else: - from io import StringIO - from os import path import yaml from kubernetes import client +if sys.version_info.major < 3: + from StringIO import StringIO # noqa: F406 +else: + from io import StringIO # noqa: F406 + def create_from_yaml( k8s_client, From f6566ee5e6ca5d743fd81f8d216079385a3dbca4 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 20 Jun 2019 13:27:33 +0200 Subject: [PATCH 10/16] Rename yml_document paramter and update documentation Renaming `yml_document` in `create_from_dict` to data. This is a bit clearer that this it a data item and not a string (usually document read from the file system). Also update the documentation to describe better what the functions `create_from_dict` and `create_from_yaml` do. --- kubernetes/utils/create_from_yaml.py | 32 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index aecc3672a6..97e39a33b9 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -39,11 +39,6 @@ def create_from_yaml( verbose: If True, print confirmation from the create action. Default is False. - Returns: - An k8s api object or list of apis objects created from YAML. - When a single object is generated, return type is dependent - on output_list. - Throws a FailToCreateError exception if creation of any object fails with helpful messages from the server. @@ -80,19 +75,34 @@ def create_from_yaml( raise FailToCreateError(fail_exceptions) -def create_from_dict(k8s_client, yml_document, verbose=False, **kwargs): +def create_from_dict(k8s_client, data, verbose=False, **kwargs): + """ + Perform an action from a dictionary containing one or more valid kubernetes + objects + + Input: + k8s_client: an ApiClient object, initialized with the client args. + data: a dictionary holding valid kubernetes objects + verbose: If True, print confirmation from the create action. + Default is False. + + Returns: + A list of `client.rest.ApiException` instances for each object that + failed to create. The user of this function can throw discard them. + + """ # If it is a list type, will need to iterate its items api_exceptions = [] - if "List" in yml_document["kind"]: + if "List" in data["kind"]: # Could be "List" or "Pod/Service/...List" # This is a list type. iterate within its items - kind = yml_document["kind"].replace("List", "") - for yml_object in yml_document["items"]: + kind = data["kind"].replace("List", "") + for yml_object in data["items"]: # Mitigate cases when server returns a xxxList object # See kubernetes-client/python#586 if kind is not "": - yml_object["apiVersion"] = yml_document["apiVersion"] + yml_object["apiVersion"] = data["apiVersion"] yml_object["kind"] = kind try: create_from_yaml_single_item( @@ -103,7 +113,7 @@ def create_from_dict(k8s_client, yml_document, verbose=False, **kwargs): # This is a single object. Call the single item method try: create_from_yaml_single_item( - k8s_client, yml_document, verbose, **kwargs) + k8s_client, data, verbose, **kwargs) except client.rest.ApiException as api_exception: api_exceptions.append(api_exception) From 4fa0e87bd71785cce49623d51fcbb81805e3fe4c Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 20 Jun 2019 14:30:27 +0200 Subject: [PATCH 11/16] Correct the documentation for create_from_dict The function can create one or more Kuberenetes objects based on the content of data. It can handle all API objects incuding `List` type, which by itself can contain more than one Kuberenetes API objects. --- kubernetes/utils/create_from_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 1fe806f7c4..ff8a6278af 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -90,8 +90,8 @@ def create_from_yaml( def create_from_dict(k8s_client, data, verbose=False, **kwargs): """ - Perform an action from a dictionary containing one or more valid kubernetes - objects + Perform an action from a dictionary containing valid kubernetes + API object (i.e. List, Service, etc). Input: k8s_client: an ApiClient object, initialized with the client args. From 5c912f986f7066a87c45315f196c4eed239061d3 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 20 Jun 2019 18:37:58 +0200 Subject: [PATCH 12/16] Throw exception from create_from_dict This is instead of doing this in create_from_yaml --- kubernetes/utils/create_from_yaml.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index ff8a6278af..20eda3ff64 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -75,17 +75,10 @@ def create_from_yaml( yaml_file = StringIO(content.decode('utf-8')) yml_document_all = yaml.safe_load_all(yaml_file) - # Load all documents from a single YAML file - fail_exceptions = [] for yml_document in yml_document_all: - exceptions = create_from_dict(k8s_client, yml_document, verbose, - **kwargs) - if exceptions: - fail_exceptions.extend(exceptions) - - if fail_exceptions: - raise FailToCreateError(fail_exceptions) + create_from_dict(k8s_client, yml_document, verbose, + **kwargs) def create_from_dict(k8s_client, data, verbose=False, **kwargs): @@ -99,10 +92,9 @@ def create_from_dict(k8s_client, data, verbose=False, **kwargs): verbose: If True, print confirmation from the create action. Default is False. - Returns: - A list of `client.rest.ApiException` instances for each object that - failed to create. The user of this function can throw discard them. - + Raises: + FailToCreateError which holds list of `client.rest.ApiException` + instances for each object that failed to create. """ # If it is a list type, will need to iterate its items api_exceptions = [] @@ -132,7 +124,7 @@ def create_from_dict(k8s_client, data, verbose=False, **kwargs): # In case we have exceptions waiting for us, raise them if api_exceptions: - return api_exceptions + raise FailToCreateError(api_exceptions) def create_from_yaml_single_item( From ed67d89f9b19545c7e88809ab6ced25af63514d4 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 20 Jun 2019 19:53:28 +0200 Subject: [PATCH 13/16] Remove un-necessary if block --- kubernetes/utils/create_from_yaml.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 20eda3ff64..b5101174ce 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -66,13 +66,12 @@ def create_from_yaml( processing of the request. Valid values are: - All: all dry run stages will be processed """ - if path.exists(yaml_file): - with open(path.abspath(yaml_file)) as f: - content = f.read() - try: - yaml_file = StringIO(content) - except TypeError: - yaml_file = StringIO(content.decode('utf-8')) + with open(path.abspath(yaml_file)) as f: + content = f.read() + try: + yaml_file = StringIO(content) + except TypeError: + yaml_file = StringIO(content.decode('utf-8')) yml_document_all = yaml.safe_load_all(yaml_file) From 9e40421bcc685d8d65b900dfeba0fec26c3cfea8 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Thu, 20 Jun 2019 22:48:35 +0200 Subject: [PATCH 14/16] create_from_yaml function deals with files only Adding the ability to deal with strings containing yaml seems to repel to much. So we stay with create_from_yaml with a bad name. This removes the need fro StringIO to wrap strings. Also note: ``` with open('foo.txt') as f: y = yaml.safe_load_all(f) for i in y: print(i) \# raises ValueError: I/O operation on closed file. ``` Hence, we indent the whole method body into the open block. with open('foo.txt') as f: y = yaml.safe_load_all(f) for i in y: print(i) --- kubernetes/utils/create_from_yaml.py | 37 +++++++++++----------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index b5101174ce..435ebdd389 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -14,18 +14,12 @@ import re -import sys from os import path import yaml from kubernetes import client -if sys.version_info.major < 3: - from StringIO import StringIO # noqa: F406 -else: - from io import StringIO # noqa: F406 - def create_from_yaml( k8s_client, @@ -47,14 +41,6 @@ def create_from_yaml( the yaml file already contains a namespace definition this parameter has no effect. - Returns: - An k8s api object or list of apis objects created from YAML. - When a single object is generated, return type is dependent - on output_list. - - Throws a FailToCreateError exception if creation of any object - fails with helpful messages from the server. - Available parameters for creating : :param async_req bool :param bool include_uninitialized: If true, partially initialized @@ -65,19 +51,24 @@ def create_from_yaml( directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed + + Raises: + FailToCreateError which holds list of `client.rest.ApiException` + instances for each object that failed to create. """ with open(path.abspath(yaml_file)) as f: - content = f.read() - try: - yaml_file = StringIO(content) - except TypeError: - yaml_file = StringIO(content.decode('utf-8')) + yml_document_all = yaml.safe_load_all(f) - yml_document_all = yaml.safe_load_all(yaml_file) + failures = [] - for yml_document in yml_document_all: - create_from_dict(k8s_client, yml_document, verbose, - **kwargs) + for yml_document in yml_document_all: + try: + create_from_dict(k8s_client, yml_document, verbose, + **kwargs) + except FailToCreateError as failure: + failures.extend(failure.api_exceptions) + if failures: + raise FailToCreateError(failures) def create_from_dict(k8s_client, data, verbose=False, **kwargs): From ab002f760d76e37ea6a808035555837923be9bb5 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Tue, 25 Jun 2019 22:31:41 +0200 Subject: [PATCH 15/16] Remove obsolete test The current PR no longer support creating from string --- kubernetes/e2e_test/test_utils.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/kubernetes/e2e_test/test_utils.py b/kubernetes/e2e_test/test_utils.py index c393ba9f5f..ca0f63912b 100644 --- a/kubernetes/e2e_test/test_utils.py +++ b/kubernetes/e2e_test/test_utils.py @@ -59,27 +59,6 @@ def test_create_apps_deployment_from_yaml(self): except ApiException: continue - def test_create_apps_deployment_from_yaml_string(self): - k8s_client = client.api_client.ApiClient(configuration=self.config) - with open(self.path_prefix + "apps-deployment-2.yaml") as f: - yaml_str = f.read() - - utils.create_from_yaml( - k8s_client, yaml_str) - - app_api = client.AppsV1beta1Api(k8s_client) - dep = app_api.read_namespaced_deployment(name="nginx-app-2", - namespace="default") - self.assertIsNotNone(dep) - while True: - try: - app_api.delete_namespaced_deployment( - name="nginx-app-2", namespace="default", - body={}) - break - except ApiException: - continue - def test_create_apps_deployment_from_yaml_obj(self): k8s_client = client.api_client.ApiClient(configuration=self.config) with open(self.path_prefix + "apps-deployment.yaml") as f: From 6100392c47512abc6078065a21c83cd61ac7ef23 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Wed, 26 Jun 2019 00:23:21 +0200 Subject: [PATCH 16/16] Add optional namespace to create_from_dict This follows up on the addition to create_from_yaml, the behavior is the same. --- kubernetes/utils/create_from_yaml.py | 29 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/kubernetes/utils/create_from_yaml.py b/kubernetes/utils/create_from_yaml.py index 435ebdd389..af2e2aa384 100644 --- a/kubernetes/utils/create_from_yaml.py +++ b/kubernetes/utils/create_from_yaml.py @@ -60,10 +60,10 @@ def create_from_yaml( yml_document_all = yaml.safe_load_all(f) failures = [] - for yml_document in yml_document_all: try: create_from_dict(k8s_client, yml_document, verbose, + namespace=namespace, **kwargs) except FailToCreateError as failure: failures.extend(failure.api_exceptions) @@ -71,7 +71,8 @@ def create_from_yaml( raise FailToCreateError(failures) -def create_from_dict(k8s_client, data, verbose=False, **kwargs): +def create_from_dict(k8s_client, data, verbose=False, namespace='default', + **kwargs): """ Perform an action from a dictionary containing valid kubernetes API object (i.e. List, Service, etc). @@ -81,6 +82,11 @@ def create_from_dict(k8s_client, data, verbose=False, **kwargs): data: a dictionary holding valid kubernetes objects verbose: If True, print confirmation from the create action. Default is False. + namespace: string. Contains the namespace to create all + resources inside. The namespace must preexist otherwise + the resource creation will fail. If the API object in + the yaml file already contains a namespace definition + this parameter has no effect. Raises: FailToCreateError which holds list of `client.rest.ApiException` @@ -101,14 +107,15 @@ def create_from_dict(k8s_client, data, verbose=False, **kwargs): yml_object["kind"] = kind try: create_from_yaml_single_item( - k8s_client, yml_object, verbose, **kwargs) + k8s_client, yml_object, verbose, namespace=namespace, + **kwargs) except client.rest.ApiException as api_exception: api_exceptions.append(api_exception) else: # This is a single object. Call the single item method try: create_from_yaml_single_item( - k8s_client, data, verbose, **kwargs) + k8s_client, data, verbose, namespace=namespace, **kwargs) except client.rest.ApiException as api_exception: api_exceptions.append(api_exception) @@ -135,17 +142,17 @@ def create_from_yaml_single_item( kind = yml_object["kind"] kind = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', kind) kind = re.sub('([a-z0-9])([A-Z])', r'\1_\2', kind).lower() - # Decide which namespace we are going to put the object in, - # if any - if "namespace" in yml_object["metadata"]: - namespace = yml_object["metadata"]["namespace"] - else: - namespace = "default" # Expect the user to create namespaced objects more often if hasattr(k8s_api, "create_namespaced_{0}".format(kind)): + # Decide which namespace we are going to put the object in, + # if any + if "namespace" in yml_object["metadata"]: + namespace = yml_object["metadata"]["namespace"] + kwargs['namespace'] = namespace resp = getattr(k8s_api, "create_namespaced_{0}".format(kind))( - body=yml_object, namespace=namespace, **kwargs) + body=yml_object, **kwargs) else: + kwargs.pop('namespace', None) resp = getattr(k8s_api, "create_{0}".format(kind))( body=yml_object, **kwargs) if verbose: