moses_tokenizer

#!/bin/bash

# Convenience script that wraps the Moses tokenizer invocation used in the Shine Training Workflow
# for the term tokenizer. Assumes you have Moses installed somewhere. Usage:
#
#     cat corpus.txt | moses_tokenizer LANG > corpus.tok.txt
#
# LANG is optional and defaults to "en".

if [[ -z $MOSES ]]; then
   echo "Where is Moses?"
   exit
fi

lang=${1:-en}

$MOSES/scripts/tokenizer/normalize-punctuation.perl -b -l $lang \
    | $MOSES/scripts/tokenizer/remove-non-printing-char.perl -b \
    | $MOSES/scripts/tokenizer/tokenizer.perl -b -q -no-escape -protected $MOSES/scripts/tokenizer/basic-protected-patterns -l $lang