Various bits around testing

1. `--basetemp` option.

The most important/useful is the use of pytest's tmpdir stuff. This
means that you can pass, for example:

`--basetemp=${HOME}/conda-tmp`

.. I added this because by default, TMPDIR gets used and that's a
different device from where I run conda. This meant that my tests
all fell back to copying, making them slower (and less like most
of our users setups, though really we need to use pytest fixtures
so we run some of our tests with `hardlink` and `copy` settings.

As well as this, you can set `CONDA_TEST_TMPDIR` as an env. var,
but really, why would you. I may remove this.

2. `CONDA_TEST_SAVE_TEMPS` env. var.

Should probably be a fixture too.
.. Set this to prevent deletion of envs created by `make_temp_env`
and scripts made by `wrap_subprocess_call`.

3. Improve the `run_command` / `wrap_subprocess_call` argument
processing a little. This allows for more natural expression of
tests in shell, batch and python, at the cost of some 'custom'
behaviour we should document:

```
When a single multiline argument is passed it is copied verbatim.
Multiline arguements that are mixed with more arguments are not
passed verbatim but are quoted according to some specific rules.
`'` and `"` are used for quoting as deemed appropriate.
```

It may end up that we'll want a way of specifying what to replace
newlines with should be for each language? invocation of
`run_command`? For example with Python we *could* replace newlines
with `;` and then quote that. Luckily Python seems happy with
multiline arguments though (at least so far, on macOS!)

conda/cli/main_run.py

@@ -3,7 +3,7 @@
 
 from logging import getLogger
 import os
-from subprocess import list2cmdline, PIPE, Popen
+from subprocess import PIPE, Popen
 import sys
 
 from ..base.context import context
@@ -14,18 +14,27 @@
 def execute(args, parser):
     on_win = sys.platform == "win32"
 
-    # What about spaces? Though it is already too late!
-    # call = " ".join(args.executable_call)
-    call = list2cmdline(args.executable_call)
+    call = args.executable_call
     prefix = args.prefix or os.getenv("CONDA_PREFIX") or context.root_prefix
+    env = os.environ.copy()
+
     script_caller, command_args = wrap_subprocess_call(on_win, context.root_prefix, prefix, call)
-    process = Popen(command_args, universal_newlines=False, stdout=PIPE, stderr=PIPE)
+    env = os.environ.copy()
+    from conda.gateways.subprocess import _subprocess_clean_env
+    _subprocess_clean_env(env, clean_python=True, clean_conda=True)
+    process = Popen(command_args, universal_newlines=False, stdout=PIPE, stderr=PIPE, env=env)
     for line in process.stdout:
         sys.stdout.write(line.decode('utf-8'))
     stdout, stderr = process.communicate()
+    if hasattr(stdout, "decode"): stdout = stdout.decode('utf-8')
+    if hasattr(stderr, "decode"): stderr = stderr.decode('utf-8')
     if process.returncode != 0:
         log = getLogger(__name__)
         log.error("Subprocess for 'conda run {}' command failed.  Stderr was:\n{}"
                   .format(call, stderr))
     if script_caller is not None:
-        rm_rf(script_caller)
+        if not 'CONDA_TEST_SAVE_TEMPS' in os.environ:
+            rm_rf(script_caller)
+        else:
+            log = getLogger(__name__)
+            log.info('CONDA_TEST_SAVE_TEMPS :: retaining main_run script_caller {}'.format(script_caller))

conda/core/envs_manager.py

@@ -5,7 +5,7 @@
 
 from errno import EACCES
 from logging import getLogger
-from os import listdir
+from os import environ, listdir
 from os.path import dirname, isdir, isfile, join, normpath
 
 from .prefix_data import PrefixData

conda/core/link.py

@@ -1107,7 +1107,10 @@ def run_script(prefix, prec, action='post-link', env_prefix=None, activate=False
         return True
     finally:
         if script_caller is not None:
-            rm_rf(script_caller)
+            if not 'CONDA_TEST_SAVE_TEMPS' in os.environ:
+                rm_rf(script_caller)
+            else:
+                log.info('CONDA_TEST_SAVE_TEMPS :: retaining run_script {}'.format(script_caller))
 
 
 def messages(prefix):

conda/gateways/subprocess.py

@@ -48,6 +48,8 @@ def subprocess_call(command, env=None, path=None, stdin=None, raise_on_error=Tru
     ACTIVE_SUBPROCESSES.add(p)
     stdin = ensure_binary(stdin) if isinstance(stdin, string_types) else stdin
     stdout, stderr = p.communicate(input=stdin)
+    if hasattr(stdout, "decode"): stdout = stdout.decode('utf-8')
+    if hasattr(stderr, "decode"): stderr = stderr.decode('utf-8')
     rc = p.returncode
     ACTIVE_SUBPROCESSES.remove(p)
     if raise_on_error and rc != 0:

conda/utils.py

@@ -237,6 +237,7 @@ def hashsum_file(path, mode='md5'):  # pragma: no cover
             h.update(chunk)
     return h.hexdigest()
 
+from subprocess import list2cmdline
 
 @memoize
 def sys_prefix_unfollowed():
@@ -260,13 +261,42 @@ def sys_prefix_unfollowed():
     return unfollowed
 
 
-def wrap_subprocess_call(on_win, root_prefix, prefix, command):
-    env = environ.copy()
+def quote_for_shell(arguments, shell=None):
+    if not shell:
+        shell = 'cmd.exe' if on_win else 'bash'
+    if shell == 'cmd.exe':
+        return list2cmdline(arguments)
+    else:
+        # If any multiline argument gets mixed with any other argument (which is true if we've
+        # arrived in this function) then we just quote it. This assumes something like:
+        # ['python', '-c', 'a\nmultiline\nprogram\n']
+        # It may make sense to allow specifying a replacement character for '\n' too? e.g. ';'
+        quoted = []
+        # This could all be replaced with some regex wizardry but that is less readable and
+        # for code like this, readability is very important.
+        for arg in arguments:
+            quote = None
+            if '"' in arg:
+                quote = "'"
+            elif "'" in arg:
+                quote = '"'
+            elif (not ' ' in arg and not '\n' in arg):
+                quote = ''
+            else:
+                quote = '"'
+            quoted.append(quote + arg + quote)
+        return ' '.join(quoted)
+
+
+def wrap_subprocess_call(on_win, root_prefix, prefix, arguments):
     tmp_prefix = abspath(join(prefix, '.tmp'))
     script_caller = None
+    multiline = False
+    if len(arguments)==1 and '\n' in arguments[0]:
+        multiline = True
     if on_win:
         comspec = environ[str('COMSPEC')]
-        conda_bat = env.get("CONDA_BAT", abspath(join(root_prefix, 'condabin', 'conda.bat')))
+        conda_bat = environ.get("CONDA_BAT", abspath(join(root_prefix, 'condabin', 'conda.bat')))
         with Utf8NamedTemporaryFile(mode='w', prefix=tmp_prefix,
                                     suffix='.bat', delete=False) as fh:
             fh.write("@FOR /F \"tokens=100\" %%F IN ('chcp') DO @SET CONDA_OLD_CHCP=%%F\n")
@@ -275,19 +305,33 @@ def wrap_subprocess_call(on_win, root_prefix, prefix, command):
             # while helpful for debugging, this gets in the way of running wrapped commands where
             #    we care about the output.
             # fh.write('echo "PATH: %PATH%\n')
-            fh.write("@" + command.encode('utf-8') + '\n')
+            if multiline:
+                # No point silencing the first line. If that's what's wanted then
+                # it needs doing for each line and the caller may as well do that.
+                fh.write(u"{0}\n".format(arguments))
+            else:
+                fh.write("@{0}\n".format(quote_for_shell(arguments)))
             fh.write('@chcp %CONDA_OLD_CHCP%>NUL\n')
             script_caller = fh.name
         command_args = [comspec, '/d', '/c', script_caller]
     else:
         shell_path = 'sh' if 'bsd' in sys.platform else 'bash'
-        conda_exe = env.get("CONDA_EXE", abspath(join(root_prefix, 'bin', 'conda')))
-        # If we ditched Python 2, we could use `encoding='utf-8'`
+        conda_exe = environ.get("CONDA_EXE", abspath(join(root_prefix, 'bin', 'conda')))
         with Utf8NamedTemporaryFile(mode='w', prefix=tmp_prefix, delete=False) as fh:
+            fh.write(u"echo \"$(\"{0}\" \"shell.posix\" \"hook\")\"\n"
+                     .format(conda_exe))
             fh.write(u"eval \"$(\"{0}\" \"shell.posix\" \"hook\")\"\n"
                      .format(conda_exe))
             fh.write(u"conda activate \"{0}\"\n".format(prefix))
-            fh.write(u"{0}".format(command))
+            if multiline:
+                # The ' '.join() is pointless since mutliline is only True when there's 1 arg
+                # still, if that were to change this would prevent breakage.
+                fh.write(u"{0}\n".format(' '.join(arguments)))
+            elif len(arguments)==1:
+
+                fh.write(u"{0}\n".format(arguments))
+            else:
+                fh.write(u"{0}\n".format(quote_for_shell(arguments)))
             script_caller = fh.name
         command_args = [shell_path, "-x", script_caller]
 

conftest.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import warnings
 
@@ -24,7 +25,35 @@ def pytest_generate_tests(metafunc):
 
 @pytest.fixture()
 def suppress_resource_warning():
-    # suppress unclosed socket warning
-    # https://github.com/kennethreitz/requests/issues/1882
+    '''
+Suppress `Unclosed Socket Warning`
+
+It seems urllib3 keeps a socket open to avoid costly recreation costs.
+
+xref: https://github.com/kennethreitz/requests/issues/1882
+'''
     if PY3:
         warnings.filterwarnings("ignore", category=ResourceWarning)
+
+
+tmpdir_in_use = None
+def get_tmpdir():
+    return tmpdir_in_use
+@pytest.fixture(autouse=True)
+def set_tmpdir(tmpdir):
+    global tmpdir_in_use
+    if not tmpdir:
+        return tmpdir_in_use
+    td = tmpdir.strpath
+    print("Setting testing tmpdir (via CONDA_TEST_TMPDIR) to {}".format(td))
+    # I do not think setting env. vars so much is sensible, even in tests.
+    # .. to make sure this never gets misconstrued for just the dirname. It
+    # is the full path to a tmpdir with additions to it by py.test including
+    # the test name.
+    assert os.sep in td
+    os.environ['CONDA_TEST_TMPDIR'] = td
+    tmpdir_in_use = td
+
+
+#@pytest.fixture(autouse=True)
+#def tmpdir_name():