[UI] Skip faulty only when more than 1 mistake is made (hedyorg#4607)

Fixes hedyorg#4404

hedy.py

@@ -2818,7 +2818,9 @@ def transpile(input_string, level, lang="en", skip_faulty=True):
         transpile_result = transpile_inner(input_string, level, lang, populate_source_map=True)
 
     except Exception as original_error:
-        if getenv('ENABLE_SKIP_FAULTY', False) and skip_faulty:
+        hedy_amount_lines = len(input_string.strip().split('\n'))
+
+        if getenv('ENABLE_SKIP_FAULTY', False) and skip_faulty and hedy_amount_lines > 1:
             if isinstance(original_error, source_map.exceptions_not_to_skip):
                 raise original_error
             try:

tests/test_level/test_level_01.py

@@ -479,15 +479,21 @@ def test_turn_left_nl(self):
         )
 
     def test_turn_with_text_gives_error(self):
-        code = "turn koekoek"
-        expected = "pass"
+        code = textwrap.dedent("""\
+        turn koekoek
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         # We test the skipping of faulty code by checking if a certain range contains an error after executing
         # The source range consists of from_line, from_column, to_line, to_column
         # we can add multiple tests to the skipped_mappings list to test multiple error mappings
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 13), hedy.exceptions.InvalidArgumentException)
+            SkippedMapping(SourceRange(1, 1, 1, 13), hedy.exceptions.InvalidArgumentException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.single_level_tester(
@@ -580,11 +586,17 @@ def test_comments_may_be_empty(self):
     # negative tests
     #
     def test_print_with_space_gives_invalid(self):
-        code = " print Hallo welkom bij Hedy!"
-        expected = "pass"
+        code = textwrap.dedent("""\
+         print Hallo welkom bij Hedy!
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 30), hedy.exceptions.InvalidSpaceException)
+            SkippedMapping(SourceRange(1, 1, 1, 30), hedy.exceptions.InvalidSpaceException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.multi_level_tester(
@@ -594,11 +606,17 @@ def test_print_with_space_gives_invalid(self):
             max_level=1)
 
     def test_ask_with_space_gives_invalid(self):
-        code = " ask Hallo welkom bij Hedy?"
-        expected = "pass"
+        code = textwrap.dedent("""\
+         ask Hallo welkom bij Hedy?
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 28), hedy.exceptions.InvalidSpaceException)
+            SkippedMapping(SourceRange(1, 1, 1, 28), hedy.exceptions.InvalidSpaceException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.multi_level_tester(
@@ -623,11 +641,17 @@ def test_lines_with_spaces_gives_invalid(self):
             max_level=3)
 
     def test_word_plus_period_gives_invalid(self):
-        code = "word."
-        expected = "pass"
+        code = textwrap.dedent("""\
+        word.
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
             SkippedMapping(SourceRange(1, 1, 1, 6), hedy.exceptions.MissingCommandException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.single_level_tester(
@@ -637,11 +661,17 @@ def test_word_plus_period_gives_invalid(self):
         )
 
     def test_non_keyword_gives_invalid(self):
-        code = "groen"
-        expected = "pass"
+        code = textwrap.dedent("""\
+        groen
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
             SkippedMapping(SourceRange(1, 1, 1, 6), hedy.exceptions.MissingCommandException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.single_level_tester(
@@ -650,6 +680,15 @@ def test_non_keyword_gives_invalid(self):
             skipped_mappings=skipped_mappings
         )
 
+    def test_one_mistake_not_skipped(self):
+        code = "prind wrong"
+
+        self.multi_level_tester(
+            code=code,
+            exception=hedy.exceptions.InvalidCommandException,
+            max_level=3
+        )
+
     def test_lonely_echo_gives_LonelyEcho(self):
         code = "echo wat dan?"
         self.single_level_tester(code, exception=hedy.exceptions.LonelyEchoException)
@@ -664,14 +703,17 @@ def test_pint_after_empty_line_gives_error_line_3(self):
         code = textwrap.dedent("""\
         print hallo
 
-        prnt hallo""")
+        prnt hallo
+        prind skipping""")
 
         expected = textwrap.dedent("""\
         print('hallo')
+        pass
         pass""")
 
         skipped_mappings = [
             SkippedMapping(SourceRange(3, 1, 3, 11), hedy.exceptions.InvalidCommandException),
+            SkippedMapping(SourceRange(4, 1, 4, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.single_level_tester(
@@ -696,18 +738,24 @@ def test_print_without_argument_gives_incomplete_2(self):
         )
 
     def test_non_keyword_with_argument_gives_invalid(self):
-        code = "aks felienne 123"
-        expected = "pass"
+        code = textwrap.dedent("""\
+        aks felienne 123
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 18), hedy.exceptions.InvalidCommandException),
+            SkippedMapping(SourceRange(1, 1, 1, 17), hedy.exceptions.InvalidCommandException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException),
         ]
 
         self.multi_level_tester(
             code=code,
             expected=expected,
             skipped_mappings=skipped_mappings,
-            extra_check_function=lambda c: c.arguments['invalid_command'] == 'aks',
+            extra_check_function=lambda c: c.arguments['invalid_command'] in ['aks', 'prind'],
             max_level=5,
         )
 

tests/test_level/test_level_02.py

@@ -369,14 +369,17 @@ def test_one_turn_with_left_or_right_gives_type_error(self, arg):
     def test_access_before_assign_not_allowed(self):
         code = textwrap.dedent("""\
         print the name program
+        prind skipping
         name is Hedy""")
 
         expected = textwrap.dedent("""\
         pass
+        pass
         name = 'Hedy'""")
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 23), hedy.exceptions.AccessBeforeAssignException)
+            SkippedMapping(SourceRange(1, 1, 1, 23), hedy.exceptions.AccessBeforeAssignException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException)
         ]
 
         self.multi_level_tester(
@@ -416,11 +419,17 @@ def test_turn_with_non_ascii_var(self):
 
     # issue #792
     def test_turn_right_number_gives_type_error(self):
-        code = "turn right 90"
-        expected = "pass"
+        code = textwrap.dedent("""\
+        turn right 90
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 15), hedy.exceptions.InvalidArgumentException),
+            SkippedMapping(SourceRange(1, 1, 1, 14), hedy.exceptions.InvalidArgumentException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException)
         ]
 
         self.multi_level_tester(
@@ -559,11 +568,17 @@ def test_sleep_with_string_variable_gives_error(self):
     #
 
     def test_assign_with_space_gives_invalid(self):
-        code = " naam is Hedy"
-        expected = "pass"
+        code = textwrap.dedent("""\
+         naam is Hedy
+        prind skipping""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
             SkippedMapping(SourceRange(1, 1, 1, 14), hedy.exceptions.InvalidSpaceException),
+            SkippedMapping(SourceRange(2, 1, 2, 15), hedy.exceptions.InvalidCommandException)
         ]
 
         self.multi_level_tester(
@@ -769,11 +784,17 @@ def test_echo_gives_error(self):
         )
 
     def test_ask_without_var_gives_error(self):
-        code = "ask is de papier goed?"
-        expected = "pass"
+        code = textwrap.dedent("""\
+        prind skipping
+        ask is de papier goed?""")
+
+        expected = textwrap.dedent("""\
+        pass
+        pass""")
 
         skipped_mappings = [
-            SkippedMapping(SourceRange(1, 1, 1, 23), hedy.exceptions.WrongLevelException),
+            SkippedMapping(SourceRange(1, 1, 1, 15), hedy.exceptions.InvalidCommandException),
+            SkippedMapping(SourceRange(2, 1, 2, 23), hedy.exceptions.WrongLevelException),
         ]
 
         self.multi_level_tester(