Move uses of to_bytes, to_text, to_native to use the module_utils ver…

…sion (ansible#17423)

We couldn't copy to_unicode, to_bytes, to_str into module_utils because
of licensing.  So once created it we had two sets of functions that did
the same things but had different implementations.  To remedy that, this
change removes the ansible.utils.unicode versions of those functions.

bin/ansible

@@ -43,7 +43,7 @@ from multiprocessing import Lock
 import ansible.constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
 from ansible.utils.display import Display
-from ansible.utils.unicode import to_unicode
+from ansible.module_utils._text import to_text
 
 
 ########################################
@@ -97,10 +97,10 @@ if __name__ == '__main__':
 
     except AnsibleOptionsError as e:
         cli.parser.print_help()
-        display.error(to_unicode(e), wrap_text=False)
+        display.error(to_text(e), wrap_text=False)
         exit_code = 5
     except AnsibleParserError as e:
-        display.error(to_unicode(e), wrap_text=False)
+        display.error(to_text(e), wrap_text=False)
         exit_code = 4
 # TQM takes care of these, but leaving comment to reserve the exit codes
 #    except AnsibleHostUnreachable as e:
@@ -110,16 +110,16 @@ if __name__ == '__main__':
 #        display.error(str(e))
 #        exit_code = 2
     except AnsibleError as e:
-        display.error(to_unicode(e), wrap_text=False)
+        display.error(to_text(e), wrap_text=False)
         exit_code = 1
     except KeyboardInterrupt:
         display.error("User interrupted execution")
         exit_code = 99
     except Exception as e:
         have_cli_options = cli is not None and cli.options is not None
-        display.error("Unexpected Exception: %s" % to_unicode(e), wrap_text=False)
+        display.error("Unexpected Exception: %s" % to_text(e), wrap_text=False)
         if not have_cli_options or have_cli_options and cli.options.verbosity > 2:
-            display.display(u"the full traceback was:\n\n%s" % to_unicode(traceback.format_exc()))
+            display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()))
         else:
             display.display("to see the full traceback, use -vvv")
         exit_code = 250

docsite/rst/developing_modules_python3.rst

@@ -215,6 +215,28 @@ Python3.  We'll need to gather experience to see if this is going to work out
 well for modules as well or if we should give the module_utils API explicit
 switches so that modules can choose to operate with text type all of the time.
 
+Helpers
+~~~~~~~
+
+For converting between bytes, text, and native strings we have three helper
+functions.  These are :func:`ansible.module_utils._text.to_bytes`,
+:func:`ansible.module_utils._text.to_native`, and
+:func:`ansible.module_utils._text.to_text`.  These are similar to using
+``bytes.decode()`` and ``unicode.encode()`` with a few differences.
+
+* By default they try very hard not to traceback.
+* The default encoding is "utf-8"
+* There are two error strategies that don't correspond one-to-one with
+  a python codec error handler.  These are ``surrogate_or_strict`` and
+  ``surrogate_or_replace``.  ``surrogate_or_strict`` will use the ``surrogateescape``
+  error handler if available (mostly on python3) or strict if not.  It is most
+  appropriate to use when dealing with something that needs to round trip its
+  value like file paths database keys, etc.  Without ``surrogateescape`` the best
+  thing these values can do is generate a traceback that our code can catch
+  and decide how to show an error message.  ``surrogate_or_replace`` is for
+  when a value is going to be displayed to the user.  If the
+  ``surrogateescape`` error handler is not present, it will replace
+  undecodable byte sequences with a replacement character.
 
 ================================
 Porting Core Ansible to Python 3

hacking/module_formatter.py

@@ -19,6 +19,7 @@
 #
 
 from __future__ import print_function
+__metaclass__ = type
 
 import os
 import glob
@@ -34,10 +35,10 @@
 from jinja2 import Environment, FileSystemLoader
 from six import iteritems
 
+from ansible.errors import AnsibleError
+from ansible.module_utils._text import to_bytes
 from ansible.utils import module_docs
 from ansible.utils.vars import merge_hash
-from ansible.utils.unicode import to_bytes
-from ansible.errors import AnsibleError
 
 #####################################################################################
 # constants and paths

lib/ansible/cli/__init__.py

@@ -33,7 +33,7 @@
 from ansible.release import __version__
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.utils.unicode import to_bytes, to_unicode
+from ansible.module_utils._text import to_bytes, to_text
 
 try:
     from __main__ import display
@@ -109,7 +109,7 @@ def run(self):
 
         if self.options.verbosity > 0:
             if C.CONFIG_FILE:
-                display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE))
+                display.display(u"Using %s as config file" % to_text(C.CONFIG_FILE))
             else:
                 display.display(u"No config file found; using defaults")
 

lib/ansible/cli/adhoc.py

@@ -27,13 +27,13 @@
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.inventory import Inventory
+from ansible.module_utils._text import to_text
 from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
 from ansible.plugins import get_all_plugin_loaders
 from ansible.utils.vars import load_extra_vars
 from ansible.utils.vars import load_options_vars
-from ansible.utils.unicode import to_unicode
 from ansible.vars import VariableManager
 
 try:
@@ -99,7 +99,7 @@ def run(self):
         super(AdHocCLI, self).run()
 
         # only thing left should be host pattern
-        pattern = to_unicode(self.args[0], errors='strict')
+        pattern = to_text(self.args[0], errors='surrogate_or_strict')
 
         # ignore connection password cause we are local
         if self.options.connection == "local":
@@ -169,7 +169,7 @@ def run(self):
         play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval)
         play = Play().load(play_ds, variable_manager=variable_manager, loader=loader)
 
-        if self.callback: 
+        if self.callback:
             cb = self.callback
         elif self.options.one_line:
             cb = 'oneline'

lib/ansible/cli/console.py

@@ -39,18 +39,16 @@
 from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError
-
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.inventory import Inventory
+from ansible.module_utils._text import to_native, to_text
 from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
-from ansible.vars import VariableManager
+from ansible.plugins import module_loader
 from ansible.utils import module_docs
 from ansible.utils.color import stringc
-from ansible.utils.unicode import to_unicode, to_str
-from ansible.plugins import module_loader
-
+from ansible.vars import VariableManager
 
 try:
     from __main__ import display
@@ -152,11 +150,11 @@ def _find_modules_in_path(self, path):
                     continue
                 elif module.startswith('_'):
                     fullpath = '/'.join([path,module])
-                    if os.path.islink(fullpath): # avoids aliases
+                    if os.path.islink(fullpath):  # avoids aliases
                         continue
                     module = module.replace('_', '', 1)
 
-                module = os.path.splitext(module)[0] # removes the extension
+                module = os.path.splitext(module)[0]  # removes the extension
                 yield module
 
     def default(self, arg, forceshell=False):
@@ -192,11 +190,11 @@ def default(self, arg, forceshell=False):
             )
             play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader)
         except Exception as e:
-            display.error(u"Unable to build command: %s" % to_unicode(e))
+            display.error(u"Unable to build command: %s" % to_text(e))
             return False
 
         try:
-            cb = 'minimal' #FIXME: make callbacks configurable
+            cb = 'minimal'  # FIXME: make callbacks configurable
             # now create a task queue manager to execute the play
             self._tqm = None
             try:
@@ -225,8 +223,8 @@ def default(self, arg, forceshell=False):
             display.error('User interrupted execution')
             return False
         except Exception as e:
-            display.error(to_unicode(e))
-            #FIXME: add traceback in very very verbose mode
+            display.error(to_text(e))
+            # FIXME: add traceback in very very verbose mode
             return False
 
     def emptyline(self):
@@ -379,7 +377,7 @@ def complete_cd(self, text, line, begidx, endidx):
         else:
             completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)]
 
-        return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))]
+        return [to_native(s)[offs:] for s in completions if to_native(s).startswith(to_native(mline))]
 
     def completedefault(self, text, line, begidx, endidx):
         if line.split()[0] in self.modules:
@@ -394,7 +392,6 @@ def module_args(self, module_name):
         oc, a, _ = module_docs.get_docstring(in_path)
         return oc['options'].keys()
 
-
     def run(self):
 
         super(ConsoleCLI, self).run()
@@ -410,7 +407,6 @@ def run(self):
             self.pattern = self.args[0]
         self.options.cwd = self.pattern
 
-
         # dynamically add modules as commands
         self.modules = self.list_modules()
         for module in self.modules:
@@ -465,4 +461,3 @@ def run(self):
         atexit.register(readline.write_history_file, histfile)
         self.set_prompt()
         self.cmdloop()
-

lib/ansible/cli/galaxy.py

@@ -39,14 +39,15 @@
 from ansible.galaxy.login import GalaxyLogin
 from ansible.galaxy.token import GalaxyToken
 from ansible.playbook.role.requirement import RoleRequirement
-from ansible.utils.unicode import to_bytes, to_unicode
+from ansible.module_utils._text import to_bytes, to_text
 
 try:
     from __main__ import display
 except ImportError:
     from ansible.utils.display import Display
     display = Display()
 
+
 class GalaxyCLI(CLI):
 
     SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" )
@@ -65,7 +66,6 @@ def parse(self):
             epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0])
         )
 
-
         self.set_action()
 
         # common
@@ -111,7 +111,7 @@ def parse(self):
         if self.action in ['init', 'info']:
             self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles")
 
-        if not self.action in ("delete","import","init","login","setup"):
+        if self.action not in ("delete","import","init","login","setup"):
             # NOTE: while the option type=str, the default is a list, and the
             # callback will set the value to a list.
             self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH,
@@ -142,7 +142,7 @@ def exit_without_ignore(self, rc=1):
 
     def _display_role_info(self, role_info):
 
-        text = [u"", u"Role: %s" % to_unicode(role_info['name'])]
+        text = [u"", u"Role: %s" % to_text(role_info['name'])]
         text.append(u"\tdescription: %s" % role_info.get('description', ''))
 
         for k in sorted(role_info.keys()):
@@ -340,7 +340,7 @@ def execute_install(self):
                 f = open(role_file, 'r')
                 if role_file.endswith('.yaml') or role_file.endswith('.yml'):
                     try:
-                        required_roles =  yaml.safe_load(f.read())
+                        required_roles = yaml.safe_load(f.read())
                     except Exception as e:
                         raise AnsibleError("Unable to load data from the requirements file: %s" % role_file)
 
@@ -502,7 +502,7 @@ def execute_search(self):
         if len(self.args):
             terms = []
             for i in range(len(self.args)):
-               terms.append(self.args.pop())
+                terms.append(self.args.pop())
             search = '+'.join(terms[::-1])
 
         if not search and not self.options.platforms and not self.options.tags and not self.options.author:
@@ -578,8 +578,8 @@ def execute_import(self):
         if len(self.args) < 2:
             raise AnsibleError("Expected a github_username and github_repository. Use --help.")
 
-        github_repo = self.args.pop()
-        github_user = self.args.pop()
+        github_repo = to_text(self.args.pop(), errors='surrogate_or_strict')
+        github_user = to_text(self.args.pop(), errors='surrogate_or_strict')
 
         if self.options.check_status:
             task = self.api.get_import_task(github_user=github_user, github_repo=github_repo)
@@ -594,7 +594,8 @@ def execute_import(self):
                 display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED)
                 for t in task:
                     display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED)
-                display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED)
+                display.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo),
+                        color=C.COLOR_CHANGED)
                 return 0
             # found a single role as expected
             display.display("Successfully submitted import request %d" % task[0]['id'])

lib/ansible/cli/vault.py

@@ -26,7 +26,7 @@
 from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.vault import VaultEditor
 from ansible.cli import CLI
-from ansible.utils.unicode import to_unicode
+from ansible.module_utils._text import to_text
 
 try:
     from __main__ import display
@@ -163,7 +163,7 @@ def execute_view(self):
             # unicode here because we are displaying it and therefore can make
             # the decision that the display doesn't have to be precisely what
             # the input was (leave that to decrypt instead)
-            self.pager(to_unicode(self.editor.plaintext(f)))
+            self.pager(ansible.module_utils._text.to_text(self.editor.plaintext(f)))
 
     def execute_rekey(self):
         for f in self.args:

lib/ansible/errors/__init__.py

@@ -25,8 +25,7 @@
         YAML_COMMON_UNQUOTED_COLON_ERROR,
         YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR,
         YAML_COMMON_UNBALANCED_QUOTES_ERROR )
-
-from ansible.utils.unicode import to_unicode, to_str
+from ansible.module_utils._text import to_native, to_text
 
 
 class AnsibleError(Exception):
@@ -54,11 +53,11 @@ def __init__(self, message="", obj=None, show_content=True, suppress_extended_er
         if obj and isinstance(obj, AnsibleBaseYAMLObject):
             extended_error = self._get_extended_error()
             if extended_error and not suppress_extended_error:
-                self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error))
+                self.message = '%s\n\n%s' % (to_native(message), to_native(extended_error))
             else:
-                self.message = '%s' % to_str(message)
+                self.message = '%s' % to_native(message)
         else:
-            self.message = '%s' % to_str(message)
+            self.message = '%s' % to_native(message)
 
     def __str__(self):
         return self.message
@@ -104,8 +103,8 @@ def _get_extended_error(self):
             error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number)
             if src_file not in ('<string>', '<unicode>') and self._show_content:
                 (target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1)
-                target_line = to_unicode(target_line)
-                prev_line = to_unicode(prev_line)
+                target_line = to_text(target_line)
+                prev_line = to_text(prev_line)
                 if target_line:
                     stripped_line = target_line.replace(" ","")
                     arrow_line    = (" " * (col_number-1)) + "^ here"