Skip to content

Commit

Permalink
[lit] Improve readability of failing scripts.
Browse files Browse the repository at this point in the history
 - This only applies to scripts executed by the _internal_ shell script
   interpreter.

 - This patch reworks the log to look more like a shell transcript, and be less
   verbose (but in the interest of calling attention to the important parts).

Here is an example of the new format, for commands with/without failures and
with/without output:
```
$ true
$ echo hi
hi

$ false
note: command had no output on stdout or stderr
error: command failed with exit status 1

```

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@271610 91177308-0d34-0410-b5e6-96231b3b80d8
  • Loading branch information
ddunbar committed Jun 2, 2016
1 parent 25aa673 commit bf67aef
Show file tree
Hide file tree
Showing 6 changed files with 69 additions and 14 deletions.
39 changes: 31 additions & 8 deletions utils/lit/lit/TestRunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,11 +379,17 @@ def to_string(bytes):

# Ensure the resulting output is always of string type.
try:
out = to_string(out.decode('utf-8'))
if out is None:
out = ''
else:
out = to_string(out.decode('utf-8', errors='replace'))
except:
out = str(out)
try:
err = to_string(err.decode('utf-8'))
if err is None:
err = ''
else:
err = to_string(err.decode('utf-8', errors='replace'))
except:
err = str(err)

Expand Down Expand Up @@ -438,14 +444,31 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):

out = err = ''
for i,result in enumerate(results):
out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s
for s in result.command.args))
out += 'Command %d Result: %r\n' % (i, result.exitCode)
# Write the command line run.
out += '$ %s\n' % (' '.join('"%s"' % s
for s in result.command.args),)

# If nothing interesting happened, move on.
if litConfig.maxIndividualTestTime == 0 and \
result.exitCode == 0 and \
not result.stdout.strip() and not result.stderr.strip():
continue

# Otherwise, something failed or was printed, show it.
if result.stdout.strip():
out += '# command output:\n%s\n' % (result.stdout,)
if result.stderr.strip():
out += '# command stderr:\n%s\n' % (result.stderr,)
if not result.stdout.strip() and not result.stderr.strip():
out += "note: command had no output on stdout or stderr\n"

# Show the error conditions:
if result.exitCode != 0:
out += "error: command failed with exit status: %d\n" % (
result.exitCode,)
if litConfig.maxIndividualTestTime > 0:
out += 'Command %d Reached Timeout: %s\n\n' % (
out += 'error: command reached timeout: %s\n' % (
i, str(result.timeoutReached))
out += 'Command %d Output:\n%s\n\n' % (i, result.stdout)
out += 'Command %d Stderr:\n%s\n\n' % (i, result.stderr)

return out, err, exitCode, timeoutInfo

Expand Down
3 changes: 3 additions & 0 deletions utils/lit/tests/Inputs/shtest-output-printing/basic.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# RUN: true
# RUN: echo hi
# RUN: false
4 changes: 4 additions & 0 deletions utils/lit/tests/Inputs/shtest-output-printing/lit.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import lit.formats
config.name = 'shtest-output-printing'
config.suffixes = ['.txt']
config.test_format = lit.formats.ShTest(execute_external=False)
5 changes: 2 additions & 3 deletions utils/lit/tests/shtest-format.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,8 @@
#
# CHECK: Command Output (stdout):
# CHECK-NEXT: --
# CHECK-NEXT: Command 0: "printf"
# CHECK-NEXT: Command 0 Result: 0
# CHECK-NEXT: Command 0 Output:
# CHECK-NEXT: $ "printf"
# CHECK-NEXT: # command output:
# CHECK-NEXT: line 1: failed test output on stdout
# CHECK-NEXT: line 2: failed test output on stdout

Expand Down
26 changes: 26 additions & 0 deletions utils/lit/tests/shtest-output-printing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Check the various features of the ShTest format.
#
# RUN: not %{lit} -j 1 -v %{inputs}/shtest-output-printing > %t.out
# RUN: FileCheck < %t.out %s
#
# END.

# CHECK: -- Testing:

# CHECK: FAIL: shtest-output-printing :: basic.txt
# CHECK-NEXT: *** TEST 'shtest-output-printing :: basic.txt' FAILED ***
# CHECK-NEXT: Script:
# CHECK-NEXT: --
# CHECK: --
# CHECK-NEXT: Exit Code: 1
#
# CHECK: Command Output
# CHECK-NEXT: --
# CHECK-NEXT: $ "true"
# CHECK-NEXT: $ "echo" "hi"
# CHECK-NEXT: # command output:
# CHECK-NEXT: hi
#
# CHECK: $ "false"
# CHECK-NEXT: note: command had no output on stdout or stderr
# CHECK-NEXT: error: command failed with exit status: 1
6 changes: 3 additions & 3 deletions utils/lit/tests/shtest-shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@

# CHECK: FAIL: shtest-shell :: error-0.txt
# CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED ***
# CHECK: Command 0: "not-a-real-command"
# CHECK: Command 0 Result: 127
# CHECK: Command 0 Stderr:
# CHECK: $ "not-a-real-command"
# CHECK: # command stderr:
# CHECK: 'not-a-real-command': command not found
# CHECK: error: command failed with exit status: 127
# CHECK: ***

# FIXME: The output here sucks.
Expand Down

0 comments on commit bf67aef

Please sign in to comment.