Fixes so that tests pass on Python3 as well

run_tests.py

@@ -1,6 +1,13 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *  # noqa
+
 import platform
 import os
 import subprocess
@@ -88,8 +95,8 @@ def CompleterType( value ):
   if value in COMPLETERS:
     return value
   else:
-    aliases_to_completer = dict( (i,k) for k,v in COMPLETERS.iteritems()
-                                          for i in v[ 'aliases' ] )
+    aliases_to_completer = dict( ( i, k ) for k, v in COMPLETERS.items()
+                                 for i in v[ 'aliases' ] )
     if value in aliases_to_completer:
       return aliases_to_completer[ value ];
     else:

ycmd/completers/cpp/clang_completer.py

@@ -222,7 +222,7 @@ def _ResponseForInclude( self, request_data ):
     if not include_file_name:
       return None
 
-    current_file_path = ToUtf8IfNeeded( request_data[ 'filepath' ] )
+    current_file_path = request_data[ 'filepath' ]
     client_data = request_data.get( 'extra_conf_data', None )
     quoted_include_paths, include_paths = (
             self._flags.UserIncludePaths( current_file_path, client_data ) )
@@ -371,7 +371,7 @@ def DebugInfo( self, request_data ):
 
 
   def _FlagsForRequest( self, request_data ):
-    filename = ToUtf8IfNeeded( request_data[ 'filepath' ] )
+    filename = request_data[ 'filepath' ]
     if 'compilation_flags' in request_data:
       return PrepareFlagsForClang( request_data[ 'compilation_flags' ],
                                    filename )

ycmd/completers/cpp/flags.py

@@ -150,7 +150,6 @@ def Clear( self ):
 
 
 def _CallExtraConfFlagsForFile( module, filename, client_data ):
-  filename = ToUtf8IfNeeded( filename )
   # For the sake of backwards compatibility, we need to first check whether the
   # FlagsForFile function in the extra conf module even allows keyword args.
   if inspect.getargspec( module.FlagsForFile ).keywords:

ycmd/completers/cs/cs_completer.py

@@ -320,7 +320,7 @@ def _GetSolutionFile( self, filepath ):
     return self._solution_for_file[ filepath ]
 
 
-class CsharpSolutionCompleter(object):
+class CsharpSolutionCompleter( object ):
   def __init__( self, solution_path, keep_logfiles, desired_omnisharp_port ):
     self._logger = logging.getLogger( __name__ )
     self._solution_path = solution_path

ycmd/completers/cs/solutiondetection.py

@@ -29,6 +29,7 @@
 import logging
 from inspect import getfile
 from ycmd import extra_conf_store
+from ycmd.utils import ToUnicodeIfNeeded
 
 
 __logger = logging.getLogger( __name__ )
@@ -57,7 +58,7 @@ def PollModule( module, filepath ):
     try:
       module_hint = module.CSharpSolutionFile( filepath )
       __logger.info( u'extra_conf_store suggests {0} as solution file'.format(
-          str( module_hint ) ) )
+        ToUnicodeIfNeeded( module_hint ) ) )
       if module_hint:
         # received a full path or one relative to the config's location?
         candidates = [ module_hint,

ycmd/completers/general/filename_completer.py

@@ -33,7 +33,7 @@
                                               GetIncludeStatementValue )
 from ycmd.completers.cpp.clang_completer import InCFamilyFile
 from ycmd.completers.cpp.flags import Flags
-from ycmd.utils import ToUtf8IfNeeded, ToUnicodeIfNeeded, OnWindows
+from ycmd.utils import ToUnicodeIfNeeded, OnWindows
 from ycmd import responses
 
 EXTRA_INFO_MAP = { 1 : '[File]', 2 : '[Dir]', 3 : '[File&Dir]' }
@@ -102,10 +102,9 @@ def SupportedFiletypes( self ):
   def ComputeCandidatesInner( self, request_data ):
     current_line = request_data[ 'line_value' ]
     start_column = request_data[ 'start_column' ] - 1
-    orig_filepath = request_data[ 'filepath' ]
-    filetypes = request_data[ 'file_data' ][ orig_filepath ][ 'filetypes' ]
+    filepath = request_data[ 'filepath' ]
+    filetypes = request_data[ 'file_data' ][ filepath ][ 'filetypes' ]
     line = current_line[ :start_column ]
-    utf8_filepath = ToUtf8IfNeeded( orig_filepath )
 
     if InCFamilyFile( filetypes ):
       path_dir, quoted_include = (
@@ -117,7 +116,7 @@ def ComputeCandidatesInner( self, request_data ):
         return _GenerateCandidatesForPaths(
           self.GetPathsIncludeCase( path_dir,
                                     quoted_include,
-                                    utf8_filepath,
+                                    filepath,
                                     client_data ) )
 
     path_match = self._path_regex.search( line )
@@ -132,7 +131,7 @@ def ComputeCandidatesInner( self, request_data ):
       _GetPathsStandardCase(
         path_dir,
         self.user_options[ 'filepath_completion_use_working_dir' ],
-        utf8_filepath,
+        filepath,
         working_dir) )
 
 
@@ -187,7 +186,6 @@ def _GetAbsolutePathForCompletions( path_dir,
 
 
 def _GetPathsStandardCase( path_dir, use_working_dir, filepath, working_dir ):
-
   absolute_path_dir = _GetAbsolutePathForCompletions( path_dir,
                                                       use_working_dir,
                                                       filepath,

ycmd/completers/general/ultisnips_completer.py

@@ -53,8 +53,7 @@ def ComputeCandidates( self, request_data ):
   def OnBufferVisit( self, request_data ):
     raw_candidates = request_data.get( 'ultisnips_snippets', [] )
     self._candidates = [
-        responses.BuildCompletionData(
-            str( snip[ 'trigger' ] ),
-            str( '<snip> ' + snip[ 'description' ].encode( 'utf-8' ) ) )
-        for snip in raw_candidates ]
+      responses.BuildCompletionData( snip[ 'trigger' ],
+                                     '<snip> ' + snip[ 'description' ] )
+      for snip in raw_candidates ]
 

ycmd/completers/go/gocode_completer.py

@@ -30,6 +30,7 @@
 
 from ycmd import responses
 from ycmd import utils
+from ycmd.utils import ToBytes, ToUnicodeIfNeeded
 from ycmd.completers.completer import Completer
 
 GO_FILETYPES = set( [ 'go' ] )
@@ -93,18 +94,20 @@ def ComputeCandidatesInner( self, request_data ):
 
     stdoutdata = self._ExecuteBinary( self._binary_gocode,
                                       '-f=json', 'autocomplete',
-                                       filename, str(offset),
-                                       contents = contents )
+                                      filename,
+                                      str( offset ),
+                                      contents = contents )
 
     try:
-      resultdata = json.loads( stdoutdata )
+      resultdata = json.loads( ToUnicodeIfNeeded( stdoutdata ) )
     except ValueError:
       _logger.error( PARSE_ERROR_MESSAGE )
       raise RuntimeError( PARSE_ERROR_MESSAGE )
-    if len(resultdata) != 2:
+
+    if len( resultdata ) != 2:
       _logger.error( NO_COMPLETIONS_MESSAGE )
       raise RuntimeError( NO_COMPLETIONS_MESSAGE )
-    for result in resultdata[1]:
+    for result in resultdata[ 1 ]:
       if result.get( 'class' ) == "PANIC":
         raise RuntimeError( GOCODE_PANIC_MESSAGE )
 
@@ -175,14 +178,13 @@ def _ExecuteBinary( self, binary, *args, **kwargs):
     """ Execute the GoCode/GoDef binary with given arguments. Use the contents
     argument to send data to GoCode. Return the standard output. """
     popen_handle = self._popener(
-      [ binary ] + list(args), stdin = subprocess.PIPE,
+      [ binary ] + list( args ), stdin = subprocess.PIPE,
       stdout = subprocess.PIPE, stderr = subprocess.PIPE )
     contents = kwargs[ 'contents' ] if 'contents' in kwargs else None
     stdoutdata, stderrdata = popen_handle.communicate( contents )
+
     if popen_handle.returncode:
-      binary_str = "Gocode"
-      if binary == self._binary_godef:
-        binary_str = "Godef"
+      binary_str = 'Godef' if binary == self._binary_godef else 'Gocode'
 
       _logger.error( SHELL_ERROR_MESSAGE.format( binary_str ) +
                      " code %i stderr: %s",
@@ -209,12 +211,13 @@ def _GoToDefinition( self, request_data ):
                                     "-o=%s" % offset,
                                     contents = contents )
       return self._ConstructGoToFromResponse( stdout )
-    except Exception:
+    except Exception as e:
+      _logger.exception( e )
       raise RuntimeError( 'Can\'t jump to definition.' )
 
 
   def _ConstructGoToFromResponse( self, response_str ):
-    parsed = json.loads( response_str )
+    parsed = json.loads( ToUnicodeIfNeeded( response_str ) )
     if 'filename' in parsed and 'column' in parsed:
       return responses.BuildGoToResponse( parsed[ 'filename' ],
                                           int( parsed[ 'line' ] ),
@@ -227,13 +230,15 @@ def _ConstructGoToFromResponse( self, response_str ):
 # TODO(ekfriis): If this is slow, consider moving this to C++ ycm_core,
 # perhaps in RequestWrap.
 def _ComputeOffset( contents, line, col ):
+  contents = ToBytes( contents )
   curline = 1
   curcol = 1
+  newline = bytes( b'\n' )[ 0 ]
   for i, byte in enumerate( contents ):
     if curline == line and curcol == col:
       return i
     curcol += 1
-    if byte == bytes( b'\n' ):
+    if byte == newline:
       curline += 1
       curcol = 1
   _logger.error( 'GoCode completer - could not compute byte offset ' +

ycmd/completers/python/jedi_completer.py

@@ -26,13 +26,13 @@
 from future.utils import native
 standard_library.install_aliases()
 
-from ycmd.utils import ToBytes, ProcessIsRunning, ToUtf8IfNeeded
+from ycmd.utils import ToBytes, ProcessIsRunning
 from ycmd.completers.completer import Completer
 from ycmd import responses, utils, hmac_utils
+from tempfile import NamedTemporaryFile
 
 from base64 import b64encode
 import json
-import tempfile
 import logging
 import urllib.parse
 import requests
@@ -108,7 +108,7 @@ def ServerIsReady( self ):
       self._logger.debug( 'JediHTTP not running.' )
       return False
     try:
-      return bool( self._GetResponse( ToBytes( '/ready' ) ) )
+      return bool( self._GetResponse( '/ready' ) )
     except requests.exceptions.ConnectionError as e:
       self._logger.exception( e )
       return False
@@ -156,8 +156,9 @@ def _StartServer( self ):
       self._hmac_secret = self._GenerateHmacSecret()
 
       # JediHTTP will delete the secret_file after it's done reading it
-      with tempfile.NamedTemporaryFile( delete = False ) as hmac_file:
-        json.dump( { 'hmac_secret': self._hmac_secret }, hmac_file )
+      with NamedTemporaryFile( delete = False, mode = 'w+' ) as hmac_file:
+        json.dump( { 'hmac_secret': str( self._hmac_secret, 'utf8' ) },
+                   hmac_file )
         command = [ self._python_binary_path,
                     PATH_TO_JEDIHTTP,
                     '--port', str( self._jedihttp_port ),
@@ -185,7 +186,7 @@ def _GenerateHmacSecret( self ):
 
   def _GetResponse( self, handler, request_data = {} ):
     """POST JSON data to JediHTTP server and return JSON response."""
-
+    handler = ToBytes( handler )
     url = urllib.parse.urljoin( self._jedihttp_host, handler )
     parameters = self._TranslateRequestForJediHTTP( request_data )
     body = ToBytes( json.dumps( parameters ) ) if parameters else bytes()
@@ -235,7 +236,7 @@ def _TranslateRequestForJediHTTP( self, request_data ):
   def _GetExtraData( self, completion ):
       location = {}
       if completion[ 'module_path' ]:
-        location[ 'filepath' ] = ToUtf8IfNeeded( completion[ 'module_path' ] )
+        location[ 'filepath' ] = completion[ 'module_path' ]
       if completion[ 'line' ]:
         location[ 'line_num' ] = completion[ 'line' ]
       if completion[ 'column' ]:
@@ -251,15 +252,15 @@ def _GetExtraData( self, completion ):
 
   def ComputeCandidatesInner( self, request_data ):
     return [ responses.BuildCompletionData(
-                ToUtf8IfNeeded( completion[ 'name' ] ),
-                ToUtf8IfNeeded( completion[ 'description' ] ),
-                ToUtf8IfNeeded( completion[ 'docstring' ] ),
+                completion[ 'name' ],
+                completion[ 'description' ],
+                completion[ 'docstring' ],
                 extra_data = self._GetExtraData( completion ) )
              for completion in self._JediCompletions( request_data ) ]
 
 
   def _JediCompletions( self, request_data ):
-    return self._GetResponse( ToBytes( b'/completions' ),
+    return self._GetResponse( '/completions',
                               request_data )[ 'completions' ]
 
 

ycmd/completers/rust/rust_completer.py

@@ -77,7 +77,7 @@ def FindRacerdBinary( user_options ):
     if os.path.isfile( racerd_user_binary ):
       return racerd_user_binary
     else:
-      _logger.warn( 'user provided racerd_binary_path is not file' )
+      _logger.warning( 'user provided racerd_binary_path is not file' )
 
   if os.path.isfile( RACERD_BINARY ):
     return RACERD_BINARY
@@ -101,8 +101,8 @@ def __init__( self, user_options ):
     self._rust_source_path = self._GetRustSrcPath()
 
     if not self._rust_source_path:
-      _logger.warn( 'No path provided for the rustc source. Please set the '
-                    'rust_src_path option' )
+      _logger.warning( 'No path provided for the rustc source. Please set the '
+                       'rust_src_path option' )
 
     if not self._racerd:
       _logger.error( BINARY_NOT_FOUND_MESSAGE )
@@ -135,7 +135,7 @@ def SupportedFiletypes( self ):
 
 
   def _GetResponse( self, handler, request_data = None,
-                    method = ToBytes( b'POST' ) ):
+                    method = 'POST'):
     """
     Query racerd via HTTP
 
@@ -144,6 +144,8 @@ def _GetResponse( self, handler, request_data = None,
     were found.
     """
     _logger.info( 'RustCompleter._GetResponse' )
+    handler = ToBytes( handler )
+    method = ToBytes( method )
     url = urllib.parse.urljoin( self._racerd_host, handler )
     parameters = self._ConvertToRacerdRequest( request_data )
     body = ToBytes( json.dumps( parameters ) ) if parameters else bytes()
@@ -165,7 +167,7 @@ def _GetResponse( self, handler, request_data = None,
     if response.status_code is http.client.NO_CONTENT:
       return None
 
-    return response.json( encoding = 'utf8' )
+    return response.json()
 
 
   def _ExtraHeaders( self, method, handler, body ):
@@ -209,7 +211,7 @@ def _ConvertToRacerdRequest( self, request_data ):
   def _GetExtraData( self, completion ):
     location = {}
     if completion[ 'file_path' ]:
-      location[ 'filepath' ] = ToBytes( completion[ 'file_path' ] )
+      location[ 'filepath' ] = completion[ 'file_path' ]
     if completion[ 'line' ]:
       location[ 'line_num' ] = completion[ 'line' ]
     if completion[ 'column' ]:
@@ -233,15 +235,15 @@ def ComputeCandidatesInner( self, request_data ):
       return []
 
     return [ responses.BuildCompletionData(
-                insertion_text = ToBytes( completion[ 'text' ] ),
-                kind = ToBytes( completion[ 'kind' ] ),
-                extra_menu_info = ToBytes( completion[ 'context' ] ),
+                insertion_text = completion[ 'text' ],
+                kind = completion[ 'kind' ],
+                extra_menu_info = completion[ 'context' ],
                 extra_data = self._GetExtraData( completion ) )
              for completion in completions ]
 
 
   def _FetchCompletions( self, request_data ):
-    return self._GetResponse( ToBytes( b'/list_completions' ), request_data )
+    return self._GetResponse( '/list_completions', request_data )
 
 
   def _StartServer( self ):
@@ -303,7 +305,7 @@ def ServerIsReady( self ):
       _logger.debug( 'Racerd not running.' )
       return False
     try:
-      self._GetResponse( ToBytes( b'/ping' ), method = ToBytes( b'GET' ) )
+      self._GetResponse( '/ping', method = 'GET' )
       return True
     # Do NOT make this except clause more generic! If you need to catch more
     # exception types, list them all out. Having `Exception` here caused FORTY
@@ -361,7 +363,7 @@ def GetSubcommandsMap( self ):
 
   def _GoToDefinition( self, request_data ):
     try:
-      definition = self._GetResponse( ToBytes( b'/find_definition' ),
+      definition = self._GetResponse( '/find_definition',
                                       request_data )
       return responses.BuildGoToResponse( definition[ 'file_path' ],
                                           definition[ 'line' ],