Parser recovery for incomplete path expressions/patterns (FuelLabs#3654)

sway-ast/src/item/item_use.rs

@@ -41,4 +41,8 @@ pub enum UseTree {
         double_colon_token: DoubleColonToken,
         suffix: Box<UseTree>,
     },
+    // to handle parsing recovery, e.g. foo::
+    Error {
+        spans: Box<[Span]>,
+    },
 }

sway-ast/src/path.rs

@@ -5,6 +5,9 @@ pub struct PathExpr {
     pub root_opt: Option<(Option<AngleBrackets<QualifiedPathRoot>>, DoubleColonToken)>,
     pub prefix: PathExprSegment,
     pub suffix: Vec<(DoubleColonToken, PathExprSegment)>,
+    // path expression with incomplete suffix are needed to do
+    // parser recovery on inputs like foo::
+    pub incomplete_suffix: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -32,7 +35,11 @@ impl Spanned for PathExpr {
 
 impl PathExpr {
     pub fn try_into_ident(self) -> Result<Ident, PathExpr> {
-        if self.root_opt.is_none() && self.suffix.is_empty() && self.prefix.generics_opt.is_none() {
+        if self.root_opt.is_none()
+            && self.suffix.is_empty()
+            && self.prefix.generics_opt.is_none()
+            && !self.incomplete_suffix
+        {
             return Ok(self.prefix.name);
         }
         Err(self)

sway-ast/src/pattern.rs

@@ -21,6 +21,8 @@ pub enum Pattern {
         fields: Braces<Punctuated<PatternStructField, CommaToken>>,
     },
     Tuple(Parens<Punctuated<Pattern, CommaToken>>),
+    // to handle parser recovery: Error represents an incomplete Constructor
+    Error(Box<[Span]>),
 }
 
 impl Spanned for Pattern {
@@ -44,6 +46,7 @@ impl Spanned for Pattern {
             Pattern::Constructor { path, args } => Span::join(path.span(), args.span()),
             Pattern::Struct { path, fields } => Span::join(path.span(), fields.span()),
             Pattern::Tuple(pat_tuple) => pat_tuple.span(),
+            Pattern::Error(spans) => spans.iter().cloned().reduce(Span::join).unwrap(),
         }
     }
 }

sway-core/src/language/parsed/expression/scrutinee.rs

@@ -36,6 +36,10 @@ pub enum Scrutinee {
         elems: Vec<Scrutinee>,
         span: Span,
     },
+    // this is to handle parser recovery
+    Error {
+        spans: Box<[Span]>,
+    },
 }
 
 #[derive(Debug, Clone)]
@@ -60,6 +64,7 @@ impl Spanned for Scrutinee {
             Scrutinee::StructScrutinee { span, .. } => span.clone(),
             Scrutinee::EnumScrutinee { span, .. } => span.clone(),
             Scrutinee::Tuple { span, .. } => span.clone(),
+            Scrutinee::Error { spans } => spans.iter().cloned().reduce(Span::join).unwrap(),
         }
     }
 }
@@ -156,7 +161,10 @@ impl Scrutinee {
                 .iter()
                 .flat_map(|scrutinee| scrutinee.gather_approximate_typeinfo_dependencies())
                 .collect::<Vec<TypeInfo>>(),
-            Scrutinee::Literal { .. } | Scrutinee::CatchAll { .. } | Scrutinee::Variable { .. } => {
+            Scrutinee::Literal { .. }
+            | Scrutinee::CatchAll { .. }
+            | Scrutinee::Variable { .. }
+            | Scrutinee::Error { .. } => {
                 vec![]
             }
         }

sway-core/src/semantic_analysis/ast_node/expression/match_expression/typed/typed_scrutinee.rs

@@ -43,6 +43,7 @@ impl ty::TyScrutinee {
                 span,
             } => type_check_enum(ctx, call_path, *value, span),
             Scrutinee::Tuple { elems, span } => type_check_tuple(ctx, elems, span),
+            Scrutinee::Error { .. } => err(vec![], vec![]),
         }
     }
 }

sway-core/src/transform/to_parsed_lang/convert_parse_tree.rs

@@ -270,6 +270,9 @@ fn use_tree_to_use_statements(
             use_tree_to_use_statements(*suffix, is_absolute, path, ret);
             path.pop().unwrap();
         }
+        UseTree::Error { .. } => {
+            // parsing error, nothing to push to the use statements collection
+        }
     }
 }
 
@@ -1184,8 +1187,14 @@ fn expr_func_app_to_expression_kind(
         root_opt,
         prefix,
         mut suffix,
+        ..
     } = match *func {
         Expr::Path(path_expr) => path_expr,
+        Expr::Error(_) => {
+            // FIXME we can do better here and return function application expression here
+            // if there are no parsing errors in the arguments
+            return Ok(ExpressionKind::Error(Box::new([span])));
+        }
         _ => {
             let error = ConvertParseTreeError::FunctionArbitraryExpression { span: func.span() };
             return Err(handler.emit_err(error.into()));
@@ -1883,7 +1892,7 @@ fn fn_arg_to_function_parameter(
             let error = ConvertParseTreeError::ConstantPatternsNotSupportedHere { span: pat_span };
             return Err(handler.emit_err(error.into()));
         }
-        Pattern::Constructor { .. } => {
+        Pattern::Constructor { .. } | Pattern::Error(..) => {
             let error =
                 ConvertParseTreeError::ConstructorPatternsNotSupportedHere { span: pat_span };
             return Err(handler.emit_err(error.into()));
@@ -2325,6 +2334,7 @@ fn path_expr_to_call_path_binding(
         root_opt,
         prefix,
         mut suffix,
+        ..
     } = path_expr;
     let is_absolute = path_root_opt_to_bool(handler, root_opt)?;
     let (prefixes, suffix, span, type_arguments) = match suffix.pop() {
@@ -2367,6 +2377,7 @@ fn path_expr_to_call_path(
         root_opt,
         prefix,
         mut suffix,
+        ..
     } = path_expr;
     let is_absolute = path_root_opt_to_bool(handler, root_opt)?;
     let call_path = match suffix.pop() {
@@ -2562,7 +2573,7 @@ fn statement_let_to_ast_nodes(
                 let error = ConvertParseTreeError::ConstantPatternsNotSupportedHere { span };
                 return Err(handler.emit_err(error.into()));
             }
-            Pattern::Constructor { .. } => {
+            Pattern::Constructor { .. } | Pattern::Error(..) => {
                 let error = ConvertParseTreeError::ConstructorPatternsNotSupportedHere { span };
                 return Err(handler.emit_err(error.into()));
             }
@@ -2885,6 +2896,7 @@ fn pattern_to_scrutinee(handler: &Handler, pattern: Pattern) -> Result<Scrutinee
             },
             span,
         },
+        Pattern::Error(spans) => Scrutinee::Error { spans },
     };
     Ok(scrutinee)
 }
@@ -2949,6 +2961,7 @@ fn path_expr_to_ident(handler: &Handler, path_expr: PathExpr) -> Result<Ident, E
         root_opt,
         prefix,
         suffix,
+        ..
     } = path_expr;
     if root_opt.is_some() || !suffix.is_empty() {
         let error = ConvertParseTreeError::PathsNotSupportedHere { span };

sway-lsp/src/traverse/parsed_tree.rs

@@ -764,6 +764,9 @@ impl<'a> ParsedTree<'a> {
                     self.collect_scrutinee(elem);
                 }
             }
+            Scrutinee::Error { .. } => {
+                // FIXME: Left for @JoshuaBatty to use.
+            }
         }
     }
 

sway-parse/src/expr/mod.rs

@@ -15,7 +15,7 @@ use sway_ast::token::Delimiter;
 use sway_ast::{
     AbiCastArgs, CodeBlockContents, Expr, ExprArrayDescriptor, ExprStructField,
     ExprTupleDescriptor, GenericArgs, IfCondition, IfExpr, LitInt, Literal, MatchBranch,
-    MatchBranchKind, PathExprSegment, Statement, StatementLet,
+    MatchBranchKind, PathExpr, PathExprSegment, Statement, StatementLet,
 };
 use sway_error::parser_error::ParseErrorKind;
 use sway_types::{Ident, Span, Spanned};
@@ -703,7 +703,13 @@ fn parse_atom(parser: &mut Parser, ctx: ParseExprCtx) -> ParseResult<Expr> {
         || parser.peek::<DoubleColonToken>().is_some()
         || parser.peek::<Ident>().is_some()
     {
-        let path = parser.parse()?;
+        let path: PathExpr = parser.parse()?;
+        if path.incomplete_suffix {
+            // We tried parsing it as a path but we didn't succeed so we try to recover this
+            // as an unknown sort of expression. This happens, for instance, when the user
+            // types `foo::`
+            return Ok(Expr::Error([path.span()].into()));
+        }
         if !ctx.parsing_conditional {
             if let Some(fields) = Braces::try_parse(parser)? {
                 return Ok(Expr::Struct { path, fields });

sway-parse/src/item/item_use.rs

@@ -2,6 +2,7 @@ use crate::{Parse, ParseBracket, ParseResult, Parser};
 
 use sway_ast::{Braces, ItemUse, UseTree};
 use sway_error::parser_error::ParseErrorKind;
+use sway_types::Spanned;
 
 impl Parse for UseTree {
     fn parse(parser: &mut Parser) -> ParseResult<UseTree> {
@@ -23,12 +24,18 @@ impl Parse for UseTree {
             });
         }
         if let Some(double_colon_token) = parser.take() {
-            let suffix = parser.parse()?;
-            return Ok(UseTree::Path {
-                prefix: name,
-                double_colon_token,
-                suffix,
-            });
+            if let Ok(suffix) = parser.parse() {
+                return Ok(UseTree::Path {
+                    prefix: name,
+                    double_colon_token,
+                    suffix,
+                });
+            } else {
+                // parser recovery for foo::
+                return Ok(UseTree::Error {
+                    spans: Box::new([name.span(), double_colon_token.span()]),
+                });
+            }
         }
         Ok(UseTree::Name { name })
     }

sway-parse/src/path.rs

@@ -4,7 +4,7 @@ use sway_ast::keywords::{DoubleColonToken, OpenAngleBracketToken, SelfToken, Sto
 use sway_ast::{
     AngleBrackets, PathExpr, PathExprSegment, PathType, PathTypeSegment, QualifiedPathRoot,
 };
-use sway_types::Ident;
+use sway_types::{Ident, Spanned};
 
 impl Parse for PathExpr {
     fn parse(parser: &mut Parser) -> ParseResult<PathExpr> {
@@ -25,15 +25,27 @@ impl Parse for PathExpr {
                 .map(|double_colon_token| (None, double_colon_token)),
         };
         let prefix = parser.parse()?;
-        let mut suffix = Vec::new();
+        let mut suffix: Vec<(DoubleColonToken, PathExprSegment)> = Vec::new();
+        let mut incomplete_suffix = false;
         while let Some(double_colon_token) = parser.take() {
-            let segment = parser.parse()?;
-            suffix.push((double_colon_token, segment));
+            if let Ok(segment) = parser.parse() {
+                suffix.push((double_colon_token, segment));
+            } else {
+                incomplete_suffix = true;
+                // this is to make the span be `foo::` instead of just `foo`
+                let dummy_path_expr_segment = PathExprSegment {
+                    name: Ident::new(double_colon_token.span()),
+                    generics_opt: None,
+                };
+                suffix.push((double_colon_token, dummy_path_expr_segment));
+                break;
+            }
         }
         Ok(PathExpr {
             root_opt,
             prefix,
             suffix,
+            incomplete_suffix,
         })
     }
 }

sway-parse/src/pattern.rs

@@ -40,6 +40,9 @@ impl Parse for Pattern {
         }
 
         let path = parser.parse::<PathExpr>()?;
+        if path.incomplete_suffix {
+            return Ok(Pattern::Error(Box::new([path.span()])));
+        }
         if let Some(args) = Parens::try_parse(parser)? {
             return Ok(Pattern::Constructor { path, args });
         }

swayfmt/src/items/item_use/mod.rs

@@ -138,6 +138,7 @@ impl Format for UseTree {
                 )?;
                 suffix.format(formatted_code, formatter)?;
             }
+            Self::Error { .. } => {}
         }
 
         Ok(())
@@ -243,6 +244,7 @@ impl LeafSpans for UseTree {
                 collected_spans.append(&mut suffix.leaf_spans());
                 collected_spans
             }
+            UseTree::Error { spans } => spans.iter().map(|s| ByteSpan::from(s.clone())).collect(),
         }
     }
 }

swayfmt/src/utils/language/pattern.rs

@@ -115,6 +115,7 @@ impl Format for Pattern {
                     },
                 )?;
             }
+            Self::Error(..) => {}
         }
         Ok(())
     }
@@ -292,6 +293,10 @@ impl LeafSpans for Pattern {
             Pattern::Tuple(tuple) => {
                 collected_spans.append(&mut tuple.leaf_spans());
             }
+            Pattern::Error(spans) => {
+                let mut leaf_spans = spans.iter().map(|s| ByteSpan::from(s.clone())).collect();
+                collected_spans.append(&mut leaf_spans)
+            }
         }
         collected_spans
     }

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_enum/Forc.lock

@@ -0,0 +1,3 @@
+[[package]]
+name = 'recover_path_enum'
+source = 'member'

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_enum/Forc.toml

@@ -0,0 +1,6 @@
+[project]
+name = "recover_path_enum"
+authors = ["Fuel Labs <[email protected]>"]
+entry = "main.sw"
+license = "Apache-2.0"
+implicit-std = false

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_enum/src/main.sw

@@ -0,0 +1,11 @@
+contract;
+
+enum Enum {
+    C1 : (),
+}
+
+fn recover_on_path() -> Enum {
+    return Enum::;
+}
+
+fn recovery_witness() -> bool { 0 }

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_enum/test.toml

@@ -0,0 +1,4 @@
+category = "fail"
+
+# check: $()Expected an identifier.
+# check: $()Mismatched types.

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_lib_definition/Forc.lock

@@ -0,0 +1,3 @@
+[[package]]
+name = 'recover_path_lib_definition'
+source = 'member'

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_lib_definition/Forc.toml

@@ -0,0 +1,7 @@
+[project]
+name = "recover_path_lib_definition"
+authors = ["Fuel Labs <[email protected]>"]
+entry = "main.sw"
+license = "Apache-2.0"
+implicit-std = false
+

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_lib_definition/src/main.sw

@@ -0,0 +1,8 @@
+contract;
+
+fn recover_on_path() -> b256 {
+    // intended to use std::hash::sha256
+    return std::hash::(u64::max());
+}
+
+fn recovery_witness() -> bool { 0 }

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_lib_definition/test.toml

@@ -0,0 +1,5 @@
+category = "fail"
+
+# check: $()Expected an identifier.
+# not: $()functions used in applications may not be arbitrary expressions
+# check: $()Mismatched types.

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_lib_import/Forc.lock

@@ -0,0 +1,13 @@
+[[package]]
+name = 'core'
+source = 'path+from-root-4CD3636B70CDF214'
+
+[[package]]
+name = 'recover_path_lib_import'
+source = 'member'
+dependencies = ['std']
+
+[[package]]
+name = 'std'
+source = 'path+from-root-4CD3636B70CDF214'
+dependencies = ['core']

test/src/e2e_vm_tests/test_programs/should_fail/recover_path_lib_import/Forc.toml

@@ -0,0 +1,8 @@
+[project]
+name = "recover_path_lib_import"
+authors = ["Fuel Labs <[email protected]>"]
+entry = "main.sw"
+license = "Apache-2.0"
+
+[dependencies]
+std = { path = "../../../../../../sway-lib-std" }