Move type checking methods out of basic.py and add unit tests (ansibl…

…e#53687)

* Move check_type_str() out of basic.py

* Move check_type_list() out of basic.py

* Move safe_eval() out of basic.py

* Move check_type_dict() out of basic.py

* Move json importing code to common location

* Move check_type_bool() out of basic.py

* Move _check_type_int() out of basic.py

* Move _check_type_float() out of basic.py

* Move _check_type_path() out of basic.py

* Move _check_type_raw() out of basic.py

* Move _check_type_bytes() out of basic.py

* Move _check_type_bits() out of basic.py

* Create text.formatters.py

Move human_to_bytes, bytes_to_human, and _lenient_lowercase out of basic.py into text.formatters.py
Change references in modules to point to function at new location

* Move _check_type_jsonarg() out of basic.py

* Rename json related functions and put them in common.text.converters

Move formatters.py to common.text.formatters.py and update references in modules.

* Rework check_type_str()

Add allow_conversion option to make the function more self-contained.
Move the messaging back to basic.py since those error messages are more relevant to using this function in the context of AnsibleModule and not when using the function in isolation.

* Add unit tests for type checking functions

* Change _lenient_lowercase to lenient_lowercase per feedback

lib/ansible/module_utils/common/_json_compat.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import types
+import json
+
+# Detect the python-json library which is incompatible
+try:
+    if not isinstance(json.loads, types.FunctionType) or not isinstance(json.dumps, types.FunctionType):
+        raise ImportError('json.loads or json.dumps were not found in the imported json library.')
+except AttributeError:
+    raise ImportError('python-json was detected, which is incompatible.')

lib/ansible/module_utils/common/text/converters.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import datetime
+import json
+
+from itertools import repeat
+
+from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.common._collections_compat import Set
+from ansible.module_utils.six import (
+    binary_type,
+    iteritems,
+    text_type,
+)
+from ansible.module_utils.six.moves import map
+
+
+def _json_encode_fallback(obj):
+    if isinstance(obj, Set):
+        return list(obj)
+    elif isinstance(obj, datetime.datetime):
+        return obj.isoformat()
+    raise TypeError("Cannot json serialize %s" % to_native(obj))
+
+
+def jsonify(data, **kwargs):
+    for encoding in ("utf-8", "latin-1"):
+        try:
+            return json.dumps(data, encoding=encoding, default=_json_encode_fallback, **kwargs)
+        # Old systems using old simplejson module does not support encoding keyword.
+        except TypeError:
+            try:
+                new_data = container_to_text(data, encoding=encoding)
+            except UnicodeDecodeError:
+                continue
+            return json.dumps(new_data, default=_json_encode_fallback, **kwargs)
+        except UnicodeDecodeError:
+            continue
+    raise UnicodeError('Invalid unicode encoding encountered')
+
+
+def container_to_bytes(d, encoding='utf-8', errors='surrogate_or_strict'):
+    ''' Recursively convert dict keys and values to byte str
+
+        Specialized for json return because this only handles, lists, tuples,
+        and dict container types (the containers that the json module returns)
+    '''
+
+    if isinstance(d, text_type):
+        return to_bytes(d, encoding=encoding, errors=errors)
+    elif isinstance(d, dict):
+        return dict(map(container_to_bytes, iteritems(d), repeat(encoding), repeat(errors)))
+    elif isinstance(d, list):
+        return list(map(container_to_bytes, d, repeat(encoding), repeat(errors)))
+    elif isinstance(d, tuple):
+        return tuple(map(container_to_bytes, d, repeat(encoding), repeat(errors)))
+    else:
+        return d
+
+
+def container_to_text(d, encoding='utf-8', errors='surrogate_or_strict'):
+    """Recursively convert dict keys and values to byte str
+
+    Specialized for json return because this only handles, lists, tuples,
+    and dict container types (the containers that the json module returns)
+    """
+
+    if isinstance(d, binary_type):
+        # Warning, can traceback
+        return to_text(d, encoding=encoding, errors=errors)
+    elif isinstance(d, dict):
+        return dict(map(container_to_text, iteritems(d), repeat(encoding), repeat(errors)))
+    elif isinstance(d, list):
+        return list(map(container_to_text, d, repeat(encoding), repeat(errors)))
+    elif isinstance(d, tuple):
+        return tuple(map(container_to_text, d, repeat(encoding), repeat(errors)))
+    else:
+        return d

lib/ansible/module_utils/common/text/formatters.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import re
+
+from ansible.module_utils.six import iteritems
+
+SIZE_RANGES = {
+    'Y': 1 << 80,
+    'Z': 1 << 70,
+    'E': 1 << 60,
+    'P': 1 << 50,
+    'T': 1 << 40,
+    'G': 1 << 30,
+    'M': 1 << 20,
+    'K': 1 << 10,
+    'B': 1,
+}
+
+
+def lenient_lowercase(lst):
+    """Lowercase elements of a list.
+
+    If an element is not a string, pass it through untouched.
+    """
+    lowered = []
+    for value in lst:
+        try:
+            lowered.append(value.lower())
+        except AttributeError:
+            lowered.append(value)
+    return lowered
+
+
+def human_to_bytes(number, default_unit=None, isbits=False):
+    """Convert number in string format into bytes (ex: '2K' => 2048) or using unit argument.
+    example: human_to_bytes('10M') <=> human_to_bytes(10, 'M')
+    """
+    m = re.search(r'^\s*(\d*\.?\d*)\s*([A-Za-z]+)?', str(number), flags=re.IGNORECASE)
+    if m is None:
+        raise ValueError("human_to_bytes() can't interpret following string: %s" % str(number))
+    try:
+        num = float(m.group(1))
+    except Exception:
+        raise ValueError("human_to_bytes() can't interpret following number: %s (original input string: %s)" % (m.group(1), number))
+
+    unit = m.group(2)
+    if unit is None:
+        unit = default_unit
+
+    if unit is None:
+        ''' No unit given, returning raw number '''
+        return int(round(num))
+    range_key = unit[0].upper()
+    try:
+        limit = SIZE_RANGES[range_key]
+    except Exception:
+        raise ValueError("human_to_bytes() failed to convert %s (unit = %s). The suffix must be one of %s" % (number, unit, ", ".join(SIZE_RANGES.keys())))
+
+    # default value
+    unit_class = 'B'
+    unit_class_name = 'byte'
+    # handling bits case
+    if isbits:
+        unit_class = 'b'
+        unit_class_name = 'bit'
+    # check unit value if more than one character (KB, MB)
+    if len(unit) > 1:
+        expect_message = 'expect %s%s or %s' % (range_key, unit_class, range_key)
+        if range_key == 'B':
+            expect_message = 'expect %s or %s' % (unit_class, unit_class_name)
+
+        if unit_class_name in unit.lower():
+            pass
+        elif unit[1] != unit_class:
+            raise ValueError("human_to_bytes() failed to convert %s. Value is not a valid string (%s)" % (number, expect_message))
+
+    return int(round(num * limit))
+
+
+def bytes_to_human(size, isbits=False, unit=None):
+    base = 'Bytes'
+    if isbits:
+        base = 'bits'
+    suffix = ''
+
+    for suffix, limit in sorted(iteritems(SIZE_RANGES), key=lambda item: -item[1]):
+        if (unit is None and size >= limit) or unit is not None and unit.upper() == suffix[0]:
+            break
+
+    if limit != 1:
+        suffix += base[0]
+    else:
+        suffix = base
+
+    return '%.2f %s' % (size / limit, suffix)