fix parsing

ljleb · Jul 22, 2023 · 15b5e42 · 15b5e42
1 parent 9615a3c
commit 15b5e42
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 2 deletions.
diff --git a/lib_neutral_prompt/neutral_prompt_parser.py b/lib_neutral_prompt/neutral_prompt_parser.py
@@ -137,7 +137,6 @@ def parse_weight(tokens: List[str]) -> float:
 
 
 def tokenize(s: str):
-    s = re.sub(r'\s+', ' ', s).strip()
     prompt_keywords_regex = '|'.join(rf'\b{keyword}\b' for keyword in prompt_keywords)
     return [s for s in re.split(rf'(\[|\]|:|{prompt_keywords_regex})', s) if s.strip()]
 

diff --git a/lib_neutral_prompt/prompt_parser_hijack.py b/lib_neutral_prompt/prompt_parser_hijack.py
@@ -14,7 +14,7 @@
 
 # the only difference with the original `re_weight` is the prefix `\s*` in `^(\s*.*?)` (originally `^(.*?)`)
 # this makes it possible to line break AND prompts without replacing all newlines with normal spaces
-prompt_parser.re_weight = re.compile(r"^(\s*.*?)(?:\s*:\s*([-+]?(?:\d+\.?|\d*\.\d+)))?\s*$")
+prompt_parser.re_weight = re.compile(r"^((?:\s|.)*?)(?:\s*:\s*([-+]?(?:\d+\.?|\d*\.\d+)))?\s*$")
 
 
 @prompt_parser_hijacker.hijack('get_multicond_prompt_list')