Fixes for WinRM/PowerShell support in v2.

- Add support for inserting module args into PowerShell modules.  Fixes ansible#11661.
- Support Windows paths containing spaces.  Applies changes from ansible#10727 to v2.  Fixes ansible#9999.  Should also fix ansible/ansible-modules-core#944 and ansible/ansible-modules-core#1007.
- Change how execution policy is set for running remote scripts.  Applies changes from ansible#11092 to v2.  Also fixes ansible/ansible-modules-core#1776.
- Use codepage 65001 (UTF-8) for WinRM connection instead of default (CP437), convert command to UTF-8 and results from UTF-8.  Replaces changes from ansible#10024.  Fixes ansible#11198.
- Close WinRM connection when task completes.
- Use win_stat, win_file and win_copy modules instead of stat, file and copy when called from within other action plugins (only when using WinRM+PowerShell).
- Unquote Windows path arguments before passing to win_stat, win_file, win_copy and slurp modules (only when using WinRM/PowerShell).
- Check for win_ping module to determine if core modules are missing (only when using WinRM/PowerShell).
- Add stdout_lines to result from running low level commands (so stdout_lines is available when using raw/script).
- Update copy action plugin to use shell functions for joining paths and checking for trailing slash.
- Update fetch action plugin to unquote source path when using Windows paths.
- Add win_copy and win_template action plugins that inherit from copy and template.
- Support running .bat and .cmd scripts using default system encoding instead of UTF-8.
- Always send PowerShell commands as base64-encoded blobs to allow for running simple PowerShell commands via raw.
- Support running modules on Windows with interpreters other than PowerShell.
- Update integration tests to support above changes and test unicode fixes.
- Add test for win_user error from ansible/ansible-modules-core#1241 (fixed by ansible/ansible-modules-core#1774).
- Add test for additional win_stat output values (implemented by ansible/ansible-modules-core#1473).
- Add test for OS architecture and name from setup.ps1 (implemented by ansible/ansible-modules-core#1100).

All WinRM integration tests pass for me with these changes.

lib/ansible/executor/module_common.py

@@ -37,6 +37,7 @@
 REPLACER_ARGS    = "\"<<INCLUDE_ANSIBLE_MODULE_ARGS>>\""
 REPLACER_COMPLEX = "\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
 REPLACER_WINDOWS = "# POWERSHELL_COMMON"
+REPLACER_WINARGS = "<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>"
 REPLACER_VERSION = "\"<<ANSIBLE_VERSION>>\""
 
 # We could end up writing out parameters with unicode characters so we need to
@@ -65,6 +66,8 @@ def _find_snippet_imports(module_data, module_path, strip_comments):
     module_style = 'old'
     if REPLACER in module_data:
         module_style = 'new'
+    elif REPLACER_WINDOWS in module_data:
+        module_style = 'new'
     elif 'from ansible.module_utils.' in module_data:
         module_style = 'new'
     elif 'WANT_JSON' in module_data:
@@ -165,6 +168,7 @@ def modify_module(module_path, module_args, task_vars=dict(), strip_comments=Fal
         # these strings should be part of the 'basic' snippet which is required to be included
         module_data = module_data.replace(REPLACER_VERSION, repr(__version__))
         module_data = module_data.replace(REPLACER_COMPLEX, encoded_args)
+        module_data = module_data.replace(REPLACER_WINARGS, module_args_json.encode('utf-8'))
 
         if module_style == 'new':
             facility = C.DEFAULT_SYSLOG_FACILITY

lib/ansible/executor/task_executor.py

@@ -127,6 +127,13 @@ def run(self):
             return result
         except AnsibleError, e:
             return dict(failed=True, msg=to_unicode(e, nonstring='simplerepr'))
+        finally:
+            try:
+                self._connection.close()
+            except AttributeError:
+                pass
+            except Exception, e:
+                debug("error closing connection: %s" % to_unicode(e))
 
     def _get_loop_items(self):
         '''

lib/ansible/module_utils/powershell.ps1

@@ -26,8 +26,17 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-# Helper function to parse Ansible JSON arguments from a file passed as
-# the single argument to the module
+# Ansible v2 will insert the module arguments below as a string containing
+# JSON; assign them to an environment variable and redefine $args so existing
+# modules will continue to work.
+$complex_args = @'
+<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>
+'@
+Set-Content env:MODULE_COMPLEX_ARGS -Value $complex_args
+$args = @('env:MODULE_COMPLEX_ARGS')
+
+# Helper function to parse Ansible JSON arguments from a "file" passed as
+# the single argument to the module.
 # Example: $params = Parse-Args $args
 Function Parse-Args($arguments)
 {

lib/ansible/plugins/action/__init__.py

@@ -67,9 +67,29 @@ def _configure_module(self, module_name, module_args, task_vars=dict()):
 
         # Search module path(s) for named module.
         module_suffixes = getattr(self._connection, 'default_suffixes', None)
+
+        # Check to determine if PowerShell modules are supported, and apply
+        # some fixes (hacks) to module name + args.
+        if module_suffixes and '.ps1' in module_suffixes:
+            # Use Windows versions of stat/file/copy modules when called from
+            # within other action plugins.
+            if module_name in ('stat', 'file', 'copy') and self._task.action != module_name:
+                module_name = 'win_%s' % module_name
+            # Remove extra quotes surrounding path parameters before sending to module.
+            if module_name in ('win_stat', 'win_file', 'win_copy', 'slurp') and module_args and hasattr(self._connection._shell, '_unquote'):
+                for key in ('src', 'dest', 'path'):
+                    if key in module_args:
+                        module_args[key] = self._connection._shell._unquote(module_args[key])
+
         module_path = self._shared_loader_obj.module_loader.find_plugin(module_name, module_suffixes)
         if module_path is None:
-            module_path2 = self._shared_loader_obj.module_loader.find_plugin('ping', module_suffixes)
+            # Use Windows version of ping module to check module paths when
+            # using a connection that supports .ps1 suffixes.
+            if module_suffixes and '.ps1' in module_suffixes:
+                ping_module = 'win_ping'
+            else:
+                ping_module = 'ping'
+            module_path2 = self._shared_loader_obj.module_loader.find_plugin(ping_module, module_suffixes)
             if module_path2 is not None:
                 raise AnsibleError("The module %s was not found in configured module paths" % (module_name))
             else:
@@ -264,9 +284,10 @@ def _remote_checksum(self, tmp, path):
 
     def _remote_expand_user(self, path, tmp):
         ''' takes a remote path and performs tilde expansion on the remote host '''
-        if not path.startswith('~'):
+        if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~
             return path
 
+        # FIXME: Can't use os.path.sep for Windows paths.
         split_path = path.split(os.path.sep, 1)
         expand_path = split_path[0]
         if expand_path == '~':
@@ -339,6 +360,8 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None, task_var
         remote_module_path = None
         if not tmp and self._late_needs_tmp_path(tmp, module_style):
             tmp = self._make_tmp_path()
+
+        if tmp:
             remote_module_path = self._connection._shell.join_path(tmp, module_name)
 
         # FIXME: async stuff here?
@@ -456,7 +479,7 @@ def _low_level_execute_command(self, cmd, tmp, sudoable=True, in_data=None, exec
         if rc is None:
             rc = 0
 
-        return dict(rc=rc, stdout=out, stderr=err)
+        return dict(rc=rc, stdout=out, stdout_lines=out.splitlines(), stderr=err)
 
     def _get_first_available_file(self, faf, of=None, searchdir='files'):
 

lib/ansible/plugins/action/copy.py

@@ -53,7 +53,7 @@ def run(self, tmp=None, task_vars=dict()):
         # Check if the source ends with a "/"
         source_trailing_slash = False
         if source:
-            source_trailing_slash = source.endswith(os.sep)
+            source_trailing_slash = self._connection._shell.path_has_trailing_slash(source)
 
         # Define content_tempfile in case we set it after finding content populated.
         content_tempfile = None
@@ -188,7 +188,7 @@ def run(self, tmp=None, task_vars=dict()):
                     continue
 
                 # Define a remote directory that we will copy the file to.
-                tmp_src = tmp + 'source'
+                tmp_src = self._connection._shell.join_path(tmp, 'source')
 
                 if not raw:
                     self._connection.put_file(source_full, tmp_src)

lib/ansible/plugins/action/fetch.py

@@ -78,6 +78,7 @@ def run(self, tmp=None, task_vars=dict()):
 
         # calculate the destination name
         if os.path.sep not in self._connection._shell.join_path('a', ''):
+            source = self._connection._shell._unquote(source)
             source_local = source.replace('\\', '/')
         else:
             source_local = source

lib/ansible/plugins/action/win_copy.py

@@ -0,0 +1,28 @@
+# (c) 2012-2014, Michael DeHaan <[email protected]>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.copy import ActionModule as CopyActionModule
+
+# Even though CopyActionModule inherits from ActionBase, we still need to
+# directly inherit from ActionBase to appease the plugin loader.
+class ActionModule(CopyActionModule, ActionBase):
+    pass

lib/ansible/plugins/action/win_template.py

@@ -0,0 +1,28 @@
+# (c) 2012-2014, Michael DeHaan <[email protected]>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.template import ActionModule as TemplateActionModule
+
+# Even though TemplateActionModule inherits from ActionBase, we still need to
+# directly inherit from ActionBase to appease the plugin loader.
+class ActionModule(TemplateActionModule, ActionBase):
+    pass

lib/ansible/plugins/connections/winrm.py

@@ -45,7 +45,7 @@
 from ansible.plugins.connections import ConnectionBase
 from ansible.plugins import shell_loader
 from ansible.utils.path import makedirs_safe
-from ansible.utils.unicode import to_bytes
+from ansible.utils.unicode import to_bytes, to_unicode
 
 class Connection(ConnectionBase):
     '''WinRM connections over HTTP/HTTPS.'''
@@ -94,7 +94,7 @@ def _winrm_connect(self):
 
             endpoint = parse.urlunsplit((scheme, netloc, '/wsman', '', ''))
 
-            self._display.debug('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), host=self._play_context.remote_addr)
+            self._display.vvvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), host=self._play_context.remote_addr)
             protocol = Protocol(
                 endpoint,
                 transport=transport,
@@ -117,30 +117,30 @@ def _winrm_connect(self):
                         raise AnsibleError("the username/password specified for this server was incorrect")
                     elif code == 411:
                         return protocol
-                self._display.debug('WINRM CONNECTION ERROR: %s' % err_msg, host=self._play_context.remote_addr)
+                self._display.vvvvv('WINRM CONNECTION ERROR: %s' % err_msg, host=self._play_context.remote_addr)
                 continue
         if exc:
             raise AnsibleError(str(exc))
 
     def _winrm_exec(self, command, args=(), from_exec=False):
         if from_exec:
-            self._display.debug("WINRM EXEC %r %r" % (command, args), host=self._play_context.remote_addr)
+            self._display.vvvvv("WINRM EXEC %r %r" % (command, args), host=self._play_context.remote_addr)
         else:
-            self._display.debugv("WINRM EXEC %r %r" % (command, args), host=self._play_context.remote_addr)
+            self._display.vvvvvv("WINRM EXEC %r %r" % (command, args), host=self._play_context.remote_addr)
         if not self.protocol:
             self.protocol = self._winrm_connect()
         if not self.shell_id:
-            self.shell_id = self.protocol.open_shell()
+            self.shell_id = self.protocol.open_shell(codepage=65001) # UTF-8
         command_id = None
         try:
-            command_id = self.protocol.run_command(self.shell_id, command, args)
+            command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args))
             response = Response(self.protocol.get_command_output(self.shell_id, command_id))
             if from_exec:
-                self._display.debug('WINRM RESULT %r' % response, host=self._play_context.remote_addr)
+                self._display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._play_context.remote_addr)
             else:
-                self._display.debugv('WINRM RESULT %r' % response, host=self._play_context.remote_addr)
-            self._display.debugv('WINRM STDOUT %s' % response.std_out, host=self._play_context.remote_addr)
-            self._display.debugv('WINRM STDERR %s' % response.std_err, host=self._play_context.remote_addr)
+                self._display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._play_context.remote_addr)
+            self._display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._play_context.remote_addr)
+            self._display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._play_context.remote_addr)
             return response
         finally:
             if command_id:
@@ -153,34 +153,42 @@ def _connect(self):
 
     def exec_command(self, cmd, tmp_path, in_data=None, sudoable=True):
         super(Connection, self).exec_command(cmd, tmp_path, in_data=in_data, sudoable=sudoable)
-
-        cmd = to_bytes(cmd)
-        cmd_parts = shlex.split(cmd, posix=False)
+        cmd_parts = shlex.split(to_bytes(cmd), posix=False)
+        cmd_parts = map(to_unicode, cmd_parts)
+        script = None
+        cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or ''
+        # Support running .ps1 files (via script/raw).
+        if cmd_ext == '.ps1':
+            script = ' '.join(['&'] + cmd_parts)
+        # Support running .bat/.cmd files; change back to the default system encoding instead of UTF-8.
+        elif cmd_ext in ('.bat', '.cmd'):
+            script = ' '.join(['[System.Console]::OutputEncoding = [System.Text.Encoding]::Default;', '&'] + cmd_parts)
+        # Encode the command if not already encoded; supports running simple PowerShell commands via raw.
+        elif '-EncodedCommand' not in cmd_parts:
+            script = ' '.join(cmd_parts)
+        if script:
+            cmd_parts = self._shell._encode_script(script, as_list=True)
         if '-EncodedCommand' in cmd_parts:
             encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1]
-            decoded_cmd = base64.b64decode(encoded_cmd)
+            decoded_cmd = to_unicode(base64.b64decode(encoded_cmd))
             self._display.vvv("EXEC %s" % decoded_cmd, host=self._play_context.remote_addr)
         else:
             self._display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr)
-        # For script/raw support.
-        if cmd_parts and cmd_parts[0].lower().endswith('.ps1'):
-            script = self._shell._build_file_cmd(cmd_parts, quote_args=False)
-            cmd_parts = self._shell._encode_script(script, as_list=True)
         try:
             result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True)
         except Exception as e:
             traceback.print_exc()
             raise AnsibleError("failed to exec cmd %s" % cmd)
-        result.std_out = to_bytes(result.std_out)
-        result.std_err = to_bytes(result.std_err)
+        result.std_out = to_unicode(result.std_out)
+        result.std_err = to_unicode(result.std_err)
         return (result.status_code, '', result.std_out, result.std_err)
 
     def put_file(self, in_path, out_path):
         super(Connection, self).put_file(in_path, out_path)
-
-        self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
+        out_path = self._shell._unquote(out_path)
+        self._display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._play_context.remote_addr)
         if not os.path.exists(in_path):
-            raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+            raise AnsibleFileNotFound('file or module does not exist: "%s"' % in_path)
         with open(in_path) as in_file:
             in_size = os.path.getsize(in_path)
             script_template = '''
@@ -206,20 +214,20 @@ def put_file(self, in_path, out_path):
                             out_path = out_path + '.ps1'
                     b64_data = base64.b64encode(out_data)
                     script = script_template % (self._shell._escape(out_path), offset, b64_data, in_size)
-                    self._display.debug("WINRM PUT %s to %s (offset=%d size=%d)" % (in_path, out_path, offset, len(out_data)), host=self._play_context.remote_addr)
+                    self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._play_context.remote_addr)
                     cmd_parts = self._shell._encode_script(script, as_list=True)
                     result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
                     if result.status_code != 0:
-                        raise IOError(result.std_err.encode('utf-8'))
+                        raise IOError(to_unicode(result.std_err))
                 except Exception:
                     traceback.print_exc()
-                    raise AnsibleError("failed to transfer file to %s" % out_path)
+                    raise AnsibleError('failed to transfer file to "%s"' % out_path)
 
     def fetch_file(self, in_path, out_path):
         super(Connection, self).fetch_file(in_path, out_path)
-
+        in_path = self._shell._unquote(in_path)
         out_path = out_path.replace('\\', '/')
-        self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
+        self._display.vvv('FETCH "%s" TO "%s"' % (in_path, out_path), host=self._play_context.remote_addr)
         buffer_size = 2**19 # 0.5MB chunks
         makedirs_safe(os.path.dirname(out_path))
         out_file = None
@@ -248,11 +256,11 @@ def fetch_file(self, in_path, out_path):
                             Exit 1;
                         }
                     ''' % dict(buffer_size=buffer_size, path=self._shell._escape(in_path), offset=offset)
-                    self._display.debug("WINRM FETCH %s to %s (offset=%d)" % (in_path, out_path, offset), host=self._play_context.remote_addr)
+                    self._display.vvvvv('WINRM FETCH "%s" to "%s" (offset=%d)' % (in_path, out_path, offset), host=self._play_context.remote_addr)
                     cmd_parts = self._shell._encode_script(script, as_list=True)
                     result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
                     if result.status_code != 0:
-                        raise IOError(result.std_err.encode('utf-8'))
+                        raise IOError(to_unicode(result.std_err))
                     if result.std_out.strip() == '[DIR]':
                         data = None
                     else:
@@ -272,7 +280,7 @@ def fetch_file(self, in_path, out_path):
                         offset += len(data)
                 except Exception:
                     traceback.print_exc()
-                    raise AnsibleError("failed to transfer file to %s" % out_path)
+                    raise AnsibleError('failed to transfer file to "%s"' % out_path)
         finally:
             if out_file:
                 out_file.close()