deal with curly quotes

redpony · May 22, 2015 · cf61f8b · cf61f8b
1 parent d3c9c36
commit cf61f8b
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
@@ -388,15 +388,18 @@ sub deep_proc_token {
     ##### step 1: separate by punct T2 on the boundary
     my $t2 = '\`|\!|\@|\+|\=|\[|\]|\<|\>|\||\(|\)|\{|\}|\?|\"|;|●|○';
     if($line =~ s/^(($t2)+)/$1 /){
+        $line =~ s/"/“/;
 	return proc_line($line);
     }
 
     if($line =~ s/(($t2)+)$/ $1/){
+        $line =~ s/"/”/;
 	return proc_line($line);
     }	
 
     ## step 2: separate by punct T2 in any position
     if($line =~ s/(($t2)+)/ $1 /g){
+        $line =~ s/"/”/g;  # probably before punctuation char
 	return proc_line($line);
     }