Support non-utf8 simulator output in a more robust manner.

vunit/ostools.py

@@ -121,7 +121,7 @@ def __init__(self, args, cwd=None):
         LOGGER.debug("Started process with pid=%i: '%s'", self._process.pid, (" ".join(args)))
 
         self._queue = InterruptableQueue()
-        self._reader = AsynchronousFileReader(self._process.stdout, self._queue)
+        self._reader = AsynchronousFileReader(change_encoding(self._process.stdout), self._queue)
         self._reader.start()
 
     def write(self, *args, **kwargs):
@@ -305,3 +305,14 @@ def renew_path(path):
         if exists(path):
             shutil.rmtree(path)
     os.makedirs(path)
+
+
+def change_encoding(textio):
+    """
+    If Python 3 change encoding of TextIOWrapper to latin-1 ignoring decode errors
+    """
+    if isinstance(textio, io.TextIOWrapper):
+        # Python 3
+        return io.TextIOWrapper(textio.buffer, encoding='latin-1', errors="ignore")
+    else:
+        return textio

vunit/test/lint/pylintrc

@@ -36,7 +36,7 @@ load-plugins=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 #disable=
-disable=too-few-public-methods, locally-disabled, redefined-variable-type, duplicate-code
+disable=too-few-public-methods, locally-disabled, redefined-variable-type, duplicate-code, file-ignored
 
 [REPORTS]
 

vunit/test/unit/non_utf8_printer.py

@@ -0,0 +1,11 @@
+# pylint: skip-file
+from sys import stdout
+
+if hasattr(stdout, "buffer"):
+    # Python 3
+    stdout.buffer.write(b"\x87")
+else:
+    # Python 2.7
+    stdout.write(b"\x87")
+
+stdout.write("\n")

vunit/test/unit/test_ostools.py

@@ -12,6 +12,7 @@
 from unittest import TestCase
 from shutil import rmtree
 from os.path import exists, dirname, join, abspath
+import sys
 from vunit.ostools import Process, renew_path
 
 
@@ -46,7 +47,7 @@ def test_run_basic_subprocess(self):
 """)
 
         output = []
-        process = Process(["python", python_script])
+        process = Process([sys.executable, python_script])
         process.consume_output(output.append)
         self.assertEqual(output, ["foo", "bar"])
 
@@ -56,7 +57,7 @@ def test_run_error_subprocess(self):
 stdout.write("error\n")
 exit(1)
 """)
-        process = Process(["python", python_script])
+        process = Process([sys.executable, python_script])
         output = []
         self.assertRaises(Process.NonZeroExitCode,
                           process.consume_output, output.append)
@@ -67,7 +68,7 @@ def test_parses_stderr(self):
 from sys import stderr
 stderr.write("error\n")
 """)
-        process = Process(["python", python_script])
+        process = Process([sys.executable, python_script])
         output = []
         process.consume_output(output.append)
         self.assertEqual(output, ["error"])
@@ -81,7 +82,14 @@ def test_output_is_parallel(self):
 sleep(1000)
 """)
 
-        process = Process(["python", python_script])
+        process = Process([sys.executable, python_script])
         message = process.next_line()
         process.terminate()
         self.assertEqual(message, "message")
+
+    def test_non_utf8_in_output(self):
+        python_script = join(dirname(__file__), "non_utf8_printer.py")
+        output = []
+        process = Process([sys.executable, python_script])
+        process.consume_output(output.append)
+        self.assertEqual(output, [chr(0x87)])

vunit/test/unit/test_ui.py

@@ -9,6 +9,7 @@
 Acceptance test of the VUnit public interface class
 """
 
+from __future__ import print_function
 import unittest
 from string import Template
 import os

vunit/test_report.py

@@ -15,6 +15,7 @@
 import socket
 import re
 from vunit.color_printer import COLOR_PRINTER
+from vunit.ostools import read_file
 
 
 class TestReport(object):
@@ -238,8 +239,7 @@ def output(self):
         file_exists = os.path.isfile(self._output_file_name)
         is_readable = os.access(self._output_file_name, os.R_OK)
         if file_exists and is_readable:
-            with open(self._output_file_name, "r") as fread:
-                return fread.read()
+            return read_file(self._output_file_name)
         else:
             return "Failed to read output file: %s" % self._output_file_name