fix tests

crates/base-db/src/include.rs

@@ -110,6 +110,7 @@ pub(crate) fn file_includes_query(
                 } else {
                     IncludeType::TryInclude
                 };
+                //TODO: Inlining the text might fail when reporting an error on a multiline inclue.
                 let text = symbol.inline_text().trim().to_string();
                 let start: u32 = symbol.range.start().into();
                 let symbol = Symbol::new(

crates/preprocessor/src/buffer.rs

@@ -40,12 +40,14 @@ impl PreprocessorBuffer {
 
     pub fn push_symbol_no_delta(&mut self, symbol: &Symbol) {
         self.contents.push_str(&symbol.text());
-        self.source_map.push_new_range(
-            symbol.range,
-            TextRange::at(TextSize::new(self.offset), symbol.range.len()),
-        );
-        let len: u32 = symbol.range.len().into();
-        self.offset += len;
+        if !symbol.range.is_empty() {
+            // Symbols with empty ranges are expanded macros.
+            self.source_map.push_new_range(
+                symbol.range,
+                TextRange::at(TextSize::new(self.offset), symbol.range.len()),
+            );
+        }
+        self.offset += symbol.text().len() as u32;
     }
 
     pub fn push_str(&mut self, string: &str) {

crates/preprocessor/src/lib.rs

@@ -543,13 +543,7 @@ where
 
     fn process_include_directive(&mut self, symbol: &Symbol, is_try: bool) {
         let text = symbol.inline_text().trim().to_string();
-        let delta: u32 = symbol.range.len().into();
-        let symbol = Symbol::new(
-            symbol.token_kind,
-            Some(&text),
-            TextRange::at(symbol.range.start(), TextSize::new(text.len() as u32)),
-            symbol.delta,
-        );
+        let line_delta = linebreak_count(symbol.text().as_str());
 
         if let Some(path) = RE_CHEVRON.captures(&text).and_then(|c| c.get(1)) {
             match (self.include_file)(
@@ -604,8 +598,8 @@ where
             }
         };
 
-        self.buffer.push_symbol(&symbol);
-        self.buffer.push_new_lines(delta);
+        self.buffer.push_symbol(symbol);
+        self.buffer.push_new_lines(line_delta as u32);
     }
 
     fn process_negative_condition(&mut self, symbol: &Symbol) -> anyhow::Result<()> {

crates/preprocessor/src/macros.rs

@@ -252,20 +252,6 @@ where
                 context_stack.push(current_context);
                 context_stack.push(new_context);
             }
-            TokenKind::Literal(Literal::StringLiteral)
-            | TokenKind::Literal(Literal::CharLiteral) => {
-                let text = &queued_symbol.symbol.inline_text();
-                reversed_expansion_stack.push(Symbol::new(
-                    queued_symbol.symbol.token_kind,
-                    Some(text),
-                    TextRange::at(
-                        queued_symbol.symbol.range.start(),
-                        TextSize::new(text.len() as u32),
-                    ),
-                    symbol.delta.to_owned(),
-                ));
-                context_stack.push(current_context);
-            }
             TokenKind::Newline | TokenKind::LineContinuation | TokenKind::Comment(_) => {
                 context_stack.push(current_context);
             }
@@ -305,10 +291,7 @@ fn expand_non_macro_define(
         .iter()
         .enumerate()
         .map(|(i, child)| {
-            let s = QueuedSymbol::new(
-                child.to_symbol(prev_range),
-                if i == 0 { delta } else { child.delta },
-            );
+            let s = QueuedSymbol::new(child.into(), if i == 0 { delta } else { child.delta });
             s.symbol.range.clone_into(&mut prev_range);
             s
         })
@@ -347,14 +330,12 @@ fn expand_macro(
     let mut new_context = MacroContext::default();
     let mut consecutive_percent = 0;
     let mut stringize_delta = None;
-    let mut prev_range = symbol.range;
     for (i, child) in macro_
         .body
         .iter()
-        .map(|s| {
-            let s = s.to_symbol(prev_range);
-            prev_range = s.range;
-            s
+        .map(|it| {
+            let it: Symbol = it.into();
+            it
         })
         .enumerate()
     {
@@ -407,10 +388,7 @@ fn expand_macro(
                         let symbol = Symbol::new(
                             TokenKind::Literal(Literal::StringLiteral),
                             Some(&stringized),
-                            TextRange::at(
-                                symbol.range.start(),
-                                TextSize::new(stringized.len() as u32),
-                            ),
+                            TextRange::default(),
                             delta,
                         );
                         new_context.push_back(QueuedSymbol::new(symbol, delta));

crates/preprocessor/src/symbol.rs

@@ -2,7 +2,7 @@ use std::hash::Hash;
 
 use deepsize::DeepSizeOf;
 use smol_str::SmolStr;
-use sourcepawn_lexer::{Delta, Symbol, TextRange, TextSize, TokenKind};
+use sourcepawn_lexer::{Delta, Symbol, TextRange, TokenKind};
 
 /// Wrapper around `Symbol` that does not contain range information.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -36,13 +36,14 @@ impl From<Symbol> for RangeLessSymbol {
     }
 }
 
-impl RangeLessSymbol {
-    pub fn to_symbol(&self, prev_range: TextRange) -> Symbol {
-        let prev_end: u32 = prev_range.end().into();
-        let range = TextRange::at(
-            TextSize::new(prev_end.saturating_add_signed(self.delta)),
-            TextSize::new(self.text.len() as u32), // FIXME: Is this wrong?
-        );
-        Symbol::new(self.token_kind, Some(&self.text), range, self.delta)
+#[allow(clippy::from_over_into)]
+impl Into<Symbol> for &RangeLessSymbol {
+    fn into(self) -> Symbol {
+        Symbol::new(
+            self.token_kind,
+            Some(&self.text),
+            TextRange::default(),
+            self.delta,
+        )
     }
 }

crates/preprocessor/tests/main.rs

@@ -1,7 +1,7 @@
 use serde::Serialize;
 use vfs::FileId;
 
-use insta::assert_json_snapshot;
+use insta::{assert_json_snapshot, assert_snapshot};
 
 fn extend_macros(
     _macro_store: &mut MacrosMap,
@@ -14,15 +14,13 @@ fn extend_macros(
 
 #[derive(Debug, Default, Serialize)]
 struct PreprocessingResult_ {
-    preprocessed_text: String,
     vec: Vec<(u32, u32, u32, u32)>,
     expanded_symbols: Vec<(u32, u32, u32, u32, u32, u32)>,
 }
 
 impl From<PreprocessingResult> for PreprocessingResult_ {
     fn from(value: PreprocessingResult) -> Self {
         Self {
-            preprocessed_text: value.preprocessed_text().to_string(),
             vec: value
                 .source_map()
                 .source_map()
@@ -58,10 +56,27 @@ impl From<PreprocessingResult> for PreprocessingResult_ {
 #[allow(unused_macros)]
 macro_rules! assert_preproc_eq {
     ($input:expr) => {
-        assert_json_snapshot!(PreprocessingResult_::from(
-            SourcepawnPreprocessor::new(FileId::from(0), $input, &mut extend_macros)
-                .preprocess_input()
-        ));
+        let res = SourcepawnPreprocessor::new(FileId::from(0), $input, &mut extend_macros)
+            .preprocess_input();
+        assert_snapshot!(res.preprocessed_text());
+        for (u_range, s_range) in res.source_map().source_map() {
+            let u_start: u32 = u_range.start().into();
+            let u_end: u32 = u_range.end().into();
+            let s_start: u32 = s_range.start().into();
+            let s_end: u32 = s_range.end().into();
+            let u_slc = u_start as usize..u_end as usize;
+            let s_slc = s_start as usize..s_end as usize;
+            assert_eq!(
+                $input[u_slc.clone()],
+                res.preprocessed_text()[s_slc.clone()],
+                "{:?} does not map to {:?}: {} is different from {}",
+                u_range,
+                s_range,
+                $input[u_slc].to_string(),
+                res.preprocessed_text()[s_slc].to_string()
+            )
+        }
+        assert_json_snapshot!(PreprocessingResult_::from(res));
     };
 }
 

crates/preprocessor/tests/snapshots/main__define_1-2.snap

@@ -0,0 +1,28 @@
+---
+source: crates/preprocessor/tests/main.rs
+assertion_line: 215
+expression: "PreprocessingResult_ :: from(res)"
+---
+{
+  "vec": [
+    [
+      0,
+      7,
+      0,
+      7
+    ],
+    [
+      8,
+      11,
+      8,
+      11
+    ],
+    [
+      12,
+      13,
+      12,
+      13
+    ]
+  ],
+  "expanded_symbols": []
+}

crates/preprocessor/tests/snapshots/main__define_expansion_1-2.snap

@@ -0,0 +1,52 @@
+---
+source: crates/preprocessor/tests/main.rs
+assertion_line: 423
+expression: "PreprocessingResult_ :: from(res)"
+---
+{
+  "vec": [
+    [
+      0,
+      7,
+      0,
+      7
+    ],
+    [
+      8,
+      11,
+      8,
+      11
+    ],
+    [
+      12,
+      13,
+      12,
+      13
+    ],
+    [
+      14,
+      17,
+      14,
+      17
+    ],
+    [
+      18,
+      21,
+      18,
+      21
+    ],
+    [
+      22,
+      23,
+      22,
+      23
+    ],
+    [
+      27,
+      28,
+      25,
+      26
+    ]
+  ],
+  "expanded_symbols": []
+}

crates/preprocessor/tests/snapshots/main__define_expansion_2-2.snap

@@ -0,0 +1,70 @@
+---
+source: crates/preprocessor/tests/main.rs
+assertion_line: 432
+expression: "PreprocessingResult_ :: from(res)"
+---
+{
+  "vec": [
+    [
+      0,
+      7,
+      0,
+      7
+    ],
+    [
+      8,
+      11,
+      8,
+      11
+    ],
+    [
+      12,
+      18,
+      12,
+      18
+    ],
+    [
+      19,
+      23,
+      19,
+      23
+    ],
+    [
+      24,
+      27,
+      24,
+      27
+    ],
+    [
+      27,
+      28,
+      27,
+      28
+    ],
+    [
+      28,
+      30,
+      28,
+      30
+    ],
+    [
+      30,
+      31,
+      30,
+      31
+    ],
+    [
+      32,
+      33,
+      32,
+      33
+    ],
+    [
+      37,
+      38,
+      40,
+      41
+    ]
+  ],
+  "expanded_symbols": []
+}