Skip to content

Commit

Permalink
Give a meaningful error when doctest has invalid prompt (elixir-lang#…
Browse files Browse the repository at this point in the history 
…5386)

The following code will break doctest

      iex(foo@HOST)1> :foo
      :foo

but now it will give a descriptive error msg.
  • Loading branch information
@eksperimental @josevalim
eksperimental authored and josevalim committed Nov 4, 2016
1 parent 039e941 commit e0affa0
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 55 deletions.
86 changes: 50 additions & 36 deletions lib/ex_unit/lib/ex_unit/doc_test.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ defmodule ExUnit.DocTest do
defp extract_tests(line_no, doc, module) do
all_lines = String.split(doc, ~r/\n/, trim: false)
lines = adjust_indent(all_lines, line_no + 1, module)
extract_tests(lines, "", "", [], true)
extract_tests(lines, "", "", [], true, module)
end

@iex_prompt ["iex>", "iex("]
Expand All @@ -451,7 +451,8 @@ defmodule ExUnit.DocTest do
end
end

defp adjust_indent(kind, [line | rest], line_no, adjusted_lines, indent, module) when kind in [:prompt, :after_prompt] do
defp adjust_indent(kind, [line | rest], line_no, adjusted_lines, indent, module)
when kind in [:prompt, :after_prompt] do
stripped_line = strip_indent(line, indent)

case String.trim_leading(line) do
Expand Down Expand Up @@ -511,91 +512,104 @@ defmodule ExUnit.DocTest do

@fences ["```", "~~~"]

defp extract_tests([], "", "", [], _) do
defp extract_tests(lines, expr_acc, expected_acc, acc, new_test, module)
defp extract_tests([], "", "", [], _, _) do
[]
end

defp extract_tests([], "", "", acc, _) do
defp extract_tests([], "", "", acc, _, _) do
Enum.reverse(acc)
end

# End of input and we've still got a test pending.
defp extract_tests([], expr_acc, expected_acc, [test | t], _) do
defp extract_tests([], expr_acc, expected_acc, [test | rest], _, _) do
test = add_expr(test, expr_acc, expected_acc)
Enum.reverse([test | t])
Enum.reverse([test | rest])
end

# We've encountered the next test on an adjacent line. Put them into one group.
defp extract_tests([{"iex>" <> _, _} | _] = list, expr_acc, expected_acc,
[test | t], newtest) when expr_acc != "" and expected_acc != "" do
defp extract_tests([{"iex>" <> _, _} | _] = list, expr_acc, expected_acc, [test | rest], new_test, module)
when expr_acc != "" and expected_acc != "" do
test = add_expr(test, expr_acc, expected_acc)
extract_tests(list, "", "", [test | t], newtest)
extract_tests(list, "", "", [test | rest], new_test, module)
end

# Store expr_acc and start a new test case.
defp extract_tests([{"iex>" <> string, line} | lines], "", expected_acc, acc, true) do
test = %{line: line, fun_arity: nil, exprs: []}
extract_tests(lines, string, expected_acc, [test | acc], false)
defp extract_tests([{"iex>" <> string, line_no} | lines], "", expected_acc, acc, true, module) do
test = %{line: line_no, fun_arity: nil, exprs: []}
extract_tests(lines, string, expected_acc, [test | acc], false, module)
end

# Store expr_acc.
defp extract_tests([{"iex>" <> string, _} | lines], "", expected_acc, acc, false) do
extract_tests(lines, string, expected_acc, acc, false)
defp extract_tests([{"iex>" <> string, _} | lines], "", expected_acc, acc, false, module) do
extract_tests(lines, string, expected_acc, acc, false, module)
end

# Still gathering expr_acc. Synonym for the next clause.
defp extract_tests([{"iex>" <> string, _} | lines], expr_acc, expected_acc, acc, newtest) do
extract_tests(lines, expr_acc <> "\n" <> string, expected_acc, acc, newtest)
defp extract_tests([{"iex>" <> string, _} | lines], expr_acc, expected_acc, acc, new_test, module) do
extract_tests(lines, expr_acc <> "\n" <> string, expected_acc, acc, new_test, module)
end

# Still gathering expr_acc. Synonym for the previous clause.
defp extract_tests([{"...>" <> string, _} | lines], expr_acc, expected_acc, acc, newtest) when expr_acc != "" do
extract_tests(lines, expr_acc <> "\n" <> string, expected_acc, acc, newtest)
defp extract_tests([{"...>" <> string, _} | lines], expr_acc, expected_acc, acc, new_test, module)
when expr_acc != "" do
extract_tests(lines, expr_acc <> "\n" <> string, expected_acc, acc, new_test, module)
end

# Expression numbers are simply skipped.
defp extract_tests([{<<"iex(", _>> <> string, line} | lines], expr_acc, expected_acc, acc, newtest) do
extract_tests([{"iex" <> skip_iex_number(string), line} | lines], expr_acc, expected_acc, acc, newtest)
defp extract_tests([{<<"iex(", _>> <> string = line, line_no} | lines],
expr_acc, expected_acc, acc, new_test, module) do
extract_tests([{"iex" <> skip_iex_number(string, module, line_no, line), line_no} | lines],
expr_acc, expected_acc, acc, new_test, module)
end

# Expression numbers are simply skipped redux.
defp extract_tests([{<<"...(", _>> <> string, line} | lines], expr_acc, expected_acc, acc, newtest) do
extract_tests([{"..." <> skip_iex_number(string), line} | lines], expr_acc, expected_acc, acc, newtest)
defp extract_tests([{<<"...(", _>> <> string, line_no} = line | lines],
expr_acc, expected_acc, acc, new_test, module) do
extract_tests([{"..." <> skip_iex_number(string, module, line_no, line), line_no} | lines],
expr_acc, expected_acc, acc, new_test, module)
end

# Skip empty or documentation line.
defp extract_tests([_ | lines], "", "", acc, _) do
extract_tests(lines, "", "", acc, true)
defp extract_tests([_ | lines], "", "", acc, _, module) do
extract_tests(lines, "", "", acc, true, module)
end

# Encountered end of fenced code block, store pending test
defp extract_tests([{<<fence::3-bytes>> <> _, _} | lines], expr_acc, expected_acc, [test | t], _)
when fence in @fences and expr_acc != "" do
defp extract_tests([{<<fence::3-bytes>> <> _, _} | lines], expr_acc, expected_acc,
[test | rest], _new_test, module)
when fence in @fences and expr_acc != "" do
test = add_expr(test, expr_acc, expected_acc)
extract_tests(lines, "", "", [test | t], true)
extract_tests(lines, "", "", [test | rest], true, module)
end

# Encountered an empty line, store pending test
defp extract_tests([{"", _} | lines], expr_acc, expected_acc, [test | t], _) do
defp extract_tests([{"", _} | lines], expr_acc, expected_acc, [test | rest], _new_test, module) do
test = add_expr(test, expr_acc, expected_acc)
extract_tests(lines, "", "", [test | t], true)
extract_tests(lines, "", "", [test | rest], true, module)
end

# Finally, parse expected_acc.
defp extract_tests([{expected, _} | lines], expr_acc, "", acc, newtest) do
extract_tests(lines, expr_acc, expected, acc, newtest)
defp extract_tests([{expected, _} | lines], expr_acc, "", acc, new_test, module) do
extract_tests(lines, expr_acc, expected, acc, new_test, module)
end

defp extract_tests([{expected, _} | lines], expr_acc, expected_acc, acc, newtest) do
extract_tests(lines, expr_acc, expected_acc <> "\n" <> expected, acc, newtest)
defp extract_tests([{expected, _} | lines], expr_acc, expected_acc, acc, new_test, module) do
extract_tests(lines, expr_acc, expected_acc <> "\n" <> expected, acc, new_test, module)
end

defp skip_iex_number(")>" <> string) do
defp skip_iex_number(")>" <> string, _module, _line_no, _line) do
">" <> string
end

defp skip_iex_number(<<_>> <> string) do
skip_iex_number(string)
defp skip_iex_number("", module, line_no, line) do
message =
"unknown IEx prompt: #{inspect line}.\nAccepted formats are: iex>, iex(1)>, ...>, ...(1)>}"
raise Error, line: line_no, module: module, message: message
end

defp skip_iex_number(<<_>> <> string, module, line_no, line) do
skip_iex_number(string, module, line_no, line)
end

defp normalize_test(%{exprs: exprs} = test, fa) do
Expand Down
61 changes: 42 additions & 19 deletions lib/ex_unit/test/ex_unit/doc_test_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,15 @@ defmodule ExUnit.DocTestTest.Numbered do
def test_fun(), do: :ok
end |> write_beam()

defmodule ExUnit.DocTestTest.Host do
@doc """
iex(foo@bar)1> 1 +
...(foo@bar)1> 2
3
"""
def test_fun(), do: :ok
end |> write_beam()

defmodule ExUnit.DocTestTest.Haiku do
@moduledoc """
This module describes the ancient Japanese poem form known as Haiku.
Expand Down Expand Up @@ -373,7 +382,7 @@ defmodule ExUnit.DocTestTest do
test "doctest failures" do
# When adding or removing lines above this line, the tests below will
# fail because we are explicitly asserting some doctest lines from
# ActuallyCompiled in the format of test/ex_unit/doc_test_test.exs:<LINE>.
# ActuallyCompiled in the format of "test/ex_unit/doc_test_test.exs:<LINE>".
defmodule ActuallyCompiled do
use ExUnit.Case
doctest ExUnit.DocTestTest.Invalid
Expand All @@ -388,7 +397,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
1) test moduledoc at ExUnit.DocTestTest.Invalid (1) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:127: syntax error before: '*'
code: 1 + * 1
stacktrace:
Expand All @@ -397,7 +406,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
2) test moduledoc at ExUnit.DocTestTest.Invalid (2) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest failed
code: 1 + hd(List.flatten([1])) === 3
left: 2
Expand All @@ -407,7 +416,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
3) test moduledoc at ExUnit.DocTestTest.Invalid (3) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest failed
code: inspect(:oops) === "#MapSet<[]>"
left: ":oops"
Expand All @@ -418,7 +427,7 @@ defmodule ExUnit.DocTestTest do
# The stacktrace points to the cause of the error
assert output =~ """
4) test moduledoc at ExUnit.DocTestTest.Invalid (4) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest failed: got UndefinedFunctionError with message "function Hello.world/0 is undefined (module Hello is not available)"
code: Hello.world
stacktrace:
Expand All @@ -428,7 +437,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
5) test moduledoc at ExUnit.DocTestTest.Invalid (5) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest failed: expected exception WhatIsThis but got RuntimeError with message "oops"
code: raise "oops"
stacktrace:
Expand All @@ -437,7 +446,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
6) test moduledoc at ExUnit.DocTestTest.Invalid (6) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest failed: wrong message for RuntimeError
expected:
"hello"
Expand All @@ -450,7 +459,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
7) test doc at ExUnit.DocTestTest.Invalid.a/0 (7) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:148: syntax error before: '*'
code: 1 + * 1
stacktrace:
Expand All @@ -459,7 +468,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
8) test doc at ExUnit.DocTestTest.Invalid.b/0 (8) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:154: syntax error before: '*'
code: 1 + * 1
stacktrace:
Expand All @@ -468,7 +477,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
9) test doc at ExUnit.DocTestTest.Invalid.dedented_past_fence/0 (9) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:178: unexpected token: "`" (column 5, codepoint U+0060)
code: 3
```
Expand All @@ -478,7 +487,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
10) test doc at ExUnit.DocTestTest.Invalid.indented_not_enough/0 (10) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:162: unexpected token: "`" (column 1, codepoint U+0060)
code: 3
`
Expand All @@ -488,7 +497,7 @@ defmodule ExUnit.DocTestTest do

assert output =~ """
11) test doc at ExUnit.DocTestTest.Invalid.indented_too_much/0 (11) (ExUnit.DocTestTest.ActuallyCompiled)
test/ex_unit/doc_test_test.exs:379
test/ex_unit/doc_test_test.exs:388
Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:170: unexpected token: "`" (column 3, codepoint U+0060)
code: 3
```
Expand All @@ -497,7 +506,7 @@ defmodule ExUnit.DocTestTest do
"""
end

test "iex prefix contains a number" do
test "IEx prefix contains a number" do
defmodule NumberedUsage do
use ExUnit.Case
doctest ExUnit.DocTestTest.Numbered
Expand All @@ -507,6 +516,20 @@ defmodule ExUnit.DocTestTest do
assert capture_io(fn -> ExUnit.run end) =~ "1 test, 0 failures"
end

test "IEx prompt contains host" do
message =
~s[unknown IEx prompt: "iex(foo@bar)1> 1 +".\nAccepted formats are: iex>, iex(1)>, ...>, ...(1)>]

regex = ~r[test/ex_unit/doc_test_test\.exs:\d+: #{Regex.escape(message)}]

assert_raise ExUnit.DocTest.Error, regex, fn ->
defmodule HostUsage do
use ExUnit.Case
doctest ExUnit.DocTestTest.Host
end
end
end

test "tags tests as doctests" do
defmodule DoctestTag do
use ExUnit.Case
Expand All @@ -532,7 +555,7 @@ defmodule ExUnit.DocTestTest do
end

test "fails on invalid module" do
assert_raise CompileError, ~r"module ExUnit.DocTestTest.Unknown is not loaded and could not be found", fn ->
assert_raise CompileError, ~r"module ExUnit\.DocTestTest\.Unknown is not loaded and could not be found", fn ->
defmodule NeverCompiled do
import ExUnit.DocTest
doctest ExUnit.DocTestTest.Unknown
Expand All @@ -541,7 +564,7 @@ defmodule ExUnit.DocTestTest do
end

test "fails when there are no docs" do
assert_raise ExUnit.DocTest.Error, ~r"could not retrieve the documentation for module ExUnit.DocTestTest", fn ->
assert_raise ExUnit.DocTest.Error, ~r"could not retrieve the documentation for module ExUnit\.DocTestTest", fn ->
defmodule NeverCompiled do
import ExUnit.DocTest
doctest ExUnit.DocTestTest
Expand All @@ -551,23 +574,23 @@ defmodule ExUnit.DocTestTest do

test "fails in indentation mismatch" do
assert_raise ExUnit.DocTest.Error,
~r[test/ex_unit/doc_test_test.exs:\d+: indentation level mismatch: " iex> bar = 2", should have been 2 spaces], fn ->
~r[test/ex_unit/doc_test_test\.exs:\d+: indentation level mismatch: " iex> bar = 2", should have been 2 spaces], fn ->
defmodule NeverCompiled do
import ExUnit.DocTest
doctest ExUnit.DocTestTest.IndentationMismatchedPrompt
end
end

assert_raise ExUnit.DocTest.Error,
~r[test/ex_unit/doc_test_test.exs:\d+: indentation level mismatch: " 3", should have been 2 spaces], fn ->
~r[test/ex_unit/doc_test_test\.exs:\d+: indentation level mismatch: " 3", should have been 2 spaces], fn ->
defmodule NeverCompiled do
import ExUnit.DocTest
doctest ExUnit.DocTestTest.IndentationTooMuch
end
end

assert_raise ExUnit.DocTest.Error,
~r[test/ex_unit/doc_test_test.exs:\d+: indentation level mismatch: \" 3\", should have been 4 spaces], fn ->
~r[test/ex_unit/doc_test_test\.exs:\d+: indentation level mismatch: \" 3\", should have been 4 spaces], fn ->
defmodule NeverCompiled do
import ExUnit.DocTest
doctest ExUnit.DocTestTest.IndentationNotEnough
Expand All @@ -577,7 +600,7 @@ defmodule ExUnit.DocTestTest do

test "fails with improper termination" do
assert_raise ExUnit.DocTest.Error,
~r[test/ex_unit/doc_test_test.exs:\d+: expected non-blank line to follow iex> prompt], fn ->
~r[test/ex_unit/doc_test_test\.exs:\d+: expected non-blank line to follow iex> prompt], fn ->
defmodule NeverCompiled do
import ExUnit.DocTest
doctest ExUnit.DocTestTest.Incomplete
Expand Down

0 comments on commit e0affa0

Please sign in to comment.