get-data.sh

set -e
#
# Data preprocessing configuration
#
CODES=30000     # number of BPE codes
N_THREADS=15    # number of threads in data preprocessing
SPLIT=${SPLIT}
DOWNSPL=100000000
#
# Read arguments
#
POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
  --langs)
    LANGS="$2"; shift 2;;
  --pairs)
    PAIRS="$2"; shift 2;;
  --reload_codes)
    RELOAD_CODES="$2"; shift 2;;
  --reload_vocab)
    RELOAD_VOCAB="$2"; shift 2;;
  --name)
    NAME="$2"; shift 2;;
  --split)
    SPLIT="$2"; shift 2;;
  --lc_no_acc)
    LC_NO_ACC="$2"; shift 2;;
  --subfolder)
    SUBFOLDER="$2"; shift 2;;
  --codes)
    CODES="$2"; shift 2;;
  --link_test)
    NOTEST=True; shift 1;;
  --link_rev)
    LINK_REV=True; shift 1;;
  --threads)
    N_THREADS="$2"; shift 2;;
  --downspl)
    DOWNSPL="$2"; shift 2;;
  --notok)
    NOTOK=True; shift 1;;
  --suffix)
    SUFFIX="$2"; shift 2;;
  *)
  POSITIONAL+=("$1")
  shift
  ;;
esac
done
set -- "${POSITIONAL[@]}"



MAIN_PATH=$PWD
TOOLS_PATH=$PWD/tools
DATA_PATH=$PWD/data
MONO_PATH=$DATA_PATH/mono/$SUBFOLDER/$NAME
PARA_PATH=$DATA_PATH/para/$SUBFOLDER
PROC_PATH=$DATA_PATH/processed/$SUBFOLDER/${NAME}$SUFFIX

# create paths
mkdir -p $TOOLS_PATH
mkdir -p $DATA_PATH
mkdir -p $MONO_PATH
mkdir -p $PARA_PATH
mkdir -p $PROC_PATH

# moses
MOSES=$TOOLS_PATH/mosesdecoder
REPLACE_UNICODE_PUNCT=$MOSES/scripts/tokenizer/replace-unicode-punctuation.perl
NORM_PUNC=$MOSES/scripts/tokenizer/normalize-punctuation.perl
REM_NON_PRINT_CHAR=$MOSES/scripts/tokenizer/remove-non-printing-char.perl
TOKENIZER=$MOSES/scripts/tokenizer/tokenizer.perl
INPUT_FROM_SGM=$MOSES/scripts/ems/support/input-from-sgm.perl

# fastBPE
FASTBPE_DIR=$TOOLS_PATH/fastBPE
FASTBPE=$TOOLS_PATH/fastBPE/fast

# Sennrich's WMT16 scripts for Romanian preprocessing
WMT16_SCRIPTS=$TOOLS_PATH/wmt16-scripts
NORMALIZE_ROMANIAN=$WMT16_SCRIPTS/preprocess/normalise-romanian.py
REMOVE_DIACRITICS=$WMT16_SCRIPTS/preprocess/remove-diacritics.py

LOWERCASE_AND_REMOVE_ACCENT="python $TOOLS_PATH/lowercase_and_remove_accent.py"

UNILANGS=`echo $LANGS | tr ',' '\n' | tr '-' '\n' | sort | uniq | tr '\n' '-' | sed 's/-$//g'`
LANGS="${LANGS//,/ }"
BPE_CODES=$PROC_PATH/codes
ALL_TOK=$MONO_PATH/${SPLIT}.all.tok
ALLSTAR_TOK=$MONO_PATH/${SPLIT}.*.tok
ALL_BPE=$PROC_PATH/${SPLIT}.all.bpe
ALLSTAR_BPE=$PROC_PATH/${SPLIT}.*.bpe
FULL_VOCAB=$PROC_PATH/vocab.$UNILANGS
echo $FULL_VOCAB
cd $MONO_PATH
for L in $LANGS; do
    if [[ $L == *"-"* ]]; then
        SRC=`echo $L | cut -f1 -d-`
        TGT=`echo $L | cut -f2 -d-`
        PAIR=.$L
    else
        SRC=$L
        TGT=
        PAIR=
    fi

    SRC_RAW=$MONO_PATH/${SPLIT}$PAIR.$SRC
    SRC_TOK=$MONO_PATH/${SPLIT}$PAIR.$SRC.tok
    TGT_RAW=$MONO_PATH/${SPLIT}$PAIR.$TGT
    TGT_TOK=$MONO_PATH/${SPLIT}$PAIR.$TGT.tok

# preprocessing commands - special case for Romanian
    if [ "$SRC" == "ro" ]; then
      SRC_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $SRC | $REM_NON_PRINT_CHAR | $NORMALIZE_ROMANIAN | $REMOVE_DIACRITICS | $TOKENIZER -l $SRC -no-escape -threads $N_THREADS"
    elif [ "$LC_NO_ACC" == "True" ]; then
      SRC_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $SRC | $REM_NON_PRINT_CHAR |             $LOWERCASE_AND_REMOVE_ACCENT | $TOKENIZER -l $SRC -no-escape -threads $N_THREADS"
    elif [ "$SRC" == "hsb" ]; then
      SRC_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $SRC | $REM_NON_PRINT_CHAR |                                            $TOKENIZER -l pl -no-escape -threads $N_THREADS"
    else
      SRC_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $SRC | $REM_NON_PRINT_CHAR |                                            $TOKENIZER -l $SRC -no-escape -threads $N_THREADS"
    fi
    if [ "$TGT" == "ro" ]; then
      TGT_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $TGT | $REM_NON_PRINT_CHAR | $NORMALIZE_ROMANIAN | $REMOVE_DIACRITICS | $TOKENIZER -l $TGT -no-escape -threads $N_THREADS"
    elif [ "$LC_NO_ACC" == "True" ]; then
      TGT_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $TGT | $REM_NON_PRINT_CHAR |             $LOWERCASE_AND_REMOVE_ACCENT | $TOKENIZER -l $TGT -no-escape -threads $N_THREADS"
    elif [ "$TGT" == "hsb" ]; then      
      TGT_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $TGT | $REM_NON_PRINT_CHAR |                                            $TOKENIZER -l pl -no-escape -threads $N_THREADS"
    else
      TGT_PREPROCESSING="$REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $TGT | $REM_NON_PRINT_CHAR |                                            $TOKENIZER -l $TGT -no-escape -threads $N_THREADS"
    fi
    
    if [[ $NOTOK == True ]]; then 
      SRC_PREPROCESSING='cat'
      TGT_PREPROCESSING='cat'
    fi

    # tokenize data
    if ! [[ -s "$SRC_TOK" ]]; then
      echo "Tokenize $SRC monolingual data..."
      eval "cat $SRC_RAW | $SRC_PREPROCESSING > $SRC_TOK"
      echo "$SRC monolingual data tokenized in: $SRC_TOK"
    fi
    if [[ ! -s "$TGT_TOK" && ! -z $TGT ]]; then
      echo "Tokenize $TGT monolingual data..."
      eval "cat $TGT_RAW | $TGT_PREPROCESSING > $TGT_TOK"
      echo "$TGT monolingual data tokenized in: $TGT_TOK"
    fi
done


cd $MAIN_PATH
if [ ! -f "$BPE_CODES" ] && [ -f "$RELOAD_CODES" ]; then
  echo "Reloading BPE codes from $RELOAD_CODES ..."
  cp $RELOAD_CODES $BPE_CODES
fi

# learn BPE codes
if [ ! -f "$BPE_CODES" ]; then
  cat $ALLSTAR_TOK > $ALL_TOK
  echo "Learning BPE codes..."
  $FASTBPE learnbpe $CODES $ALL_TOK > $BPE_CODES
  rm $ALL_TOK
fi
echo "BPE learned in $BPE_CODES"


for L in $LANGS; do
    if [[ $L == *"-"* ]]; then
        SRC=`echo $L | cut -f1 -d-`
        TGT=`echo $L | cut -f2 -d-`
        PAIR=.$L
    else
        SRC=$L
        TGT=
        PAIR=
    fi
    SRC_TRAIN_BPE=$PROC_PATH/${SPLIT}$PAIR.$SRC.bpe
    SRC_TOK=$MONO_PATH/${SPLIT}$PAIR.$SRC.tok
    TGT_TRAIN_BPE=$PROC_PATH/${SPLIT}$PAIR.$TGT.bpe
    TGT_TOK=$MONO_PATH/${SPLIT}$PAIR.$TGT.tok
    # apply BPE codes
    if ! [[ -s "$SRC_TRAIN_BPE" ]]; then
      echo "Applying $SRC BPE codes..."
      $FASTBPE applybpe $SRC_TRAIN_BPE $SRC_TOK $BPE_CODES
      echo "BPE codes applied to $SRC in: $SRC_TRAIN_BPE"
    fi
    if [[ ! -s "$TGT_TRAIN_BPE" && ! -z $TGT ]]; then
      echo "Applying $TGT BPE codes..."
      $FASTBPE applybpe $TGT_TRAIN_BPE $TGT_TOK $BPE_CODES
      echo "BPE codes applied to $TGT in: $TGT_TRAIN_BPE"
    fi
done

# reload full vocabulary
cd $MAIN_PATH
if [ ! -f "$FULL_VOCAB" ] && [ -f "$RELOAD_VOCAB" ]; then
  echo "Reloading vocabulary from $RELOAD_VOCAB ..."
  cp $RELOAD_VOCAB $FULL_VOCAB
fi

# extract full vocabulary
if ! [[ -f "$FULL_VOCAB" ]]; then
  head -q -n $DOWNSPL $ALLSTAR_BPE > $ALL_BPE
  echo "Extracting vocabulary..."
  $FASTBPE getvocab $ALL_BPE > $FULL_VOCAB
  rm $ALL_BPE
fi
echo "Full vocab in: $FULL_VOCAB"

for L in $LANGS; do
    if [[ $L == *"-"* ]]; then
        SRC=`echo $L | cut -f1 -d-`
        TGT=`echo $L | cut -f2 -d-`
        PAIR=.$L
        REVPAIR=.$TGT-$SRC
    else
        SRC=$L
        TGT=
        PAIR=
    fi
    SRC_TRAIN_BPE=$PROC_PATH/${SPLIT}$PAIR.$SRC.bpe
    TGT_TRAIN_BPE=$PROC_PATH/${SPLIT}$PAIR.$TGT.bpe
    SRC_TRAIN_BIN=$PROC_PATH/${SPLIT}$PAIR.$SRC.pth
    TGT_TRAIN_BIN=$PROC_PATH/${SPLIT}$PAIR.$TGT.pth
    SRC_TRAIN_BIN_REV=$PROC_PATH/${SPLIT}$REVPAIR.$SRC.pth
    TGT_TRAIN_BIN_REV=$PROC_PATH/${SPLIT}$REVPAIR.$TGT.pth
    # binarize data
    if ! [[ -s "$SRC_TRAIN_BIN" ]]; then
      echo "Binarizing $SRC data..."
      $MAIN_PATH/preprocess.py $FULL_VOCAB $SRC_TRAIN_BPE
      echo "$SRC binarized data in: $SRC_TRAIN_BIN"
    fi
    if [[ ! -s "$TGT_TRAIN_BIN" && ! -z $TGT ]]; then
      echo "Binarizing $TGT data..."
      $MAIN_PATH/preprocess.py $FULL_VOCAB $TGT_TRAIN_BPE
      echo "$TGT binarized data in: $TGT_TRAIN_BIN"
    fi
   
    if [[ ! -z $PAIR && $LINK_REV == True && ! -s "$SRC_TRAIN_BIN_REV" && ! -s "$TGT_TRAIN_BIN_REV" ]]; then
      cp $SRC_TRAIN_BIN $SRC_TRAIN_BIN_REV
      cp $TGT_TRAIN_BIN $TGT_TRAIN_BIN_REV
    fi


done

if [[ $NOTEST == True && $SPLIT == valid ]]; then
    for L in $LANGS; do
        if [[ $L == *"-"* ]]; then
            SRC=`echo $L | cut -f1 -d-`
            TGT=`echo $L | cut -f2 -d-`
            PAIR=.$L
            REVPAIR=.$TGT-$SRC
        else
            SRC=$L
            TGT=
            PAIR=
        fi
        SRC_TEST_BIN=$PROC_PATH/test$PAIR.$SRC.pth
        SRC_TEST_BIN_REV=$PROC_PATH/test$REVPAIR.$SRC.pth
        TGT_TEST_BIN=$PROC_PATH/test$PAIR.$TGT.pth
        TGT_TEST_BIN_REV=$PROC_PATH/test$REVPAIR.$TGT.pth
        SRC_VALID_BIN=$PROC_PATH/${SPLIT}$PAIR.$SRC.pth
        TGT_VALID_BIN=$PROC_PATH/${SPLIT}$PAIR.$TGT.pth
        if [[ ! -L $SRC_TEST_BIN ]]; then
          ln -s $SRC_VALID_BIN $SRC_TEST_BIN
        fi
          if [[ ! -s $TGT_TEST_BIN && ! -z $TGT ]]; then
            ln -s $TGT_VALID_BIN $TGT_TEST_BIN
          fi
        if [[ ! -z $PAIR && $LINK_REV == True && ! -s "$SRC_TEST_BIN_REV" && ! -s "$TGT_TEST_BIN_REV" ]]; then 
            ln -s $TGT_TEST_BIN $TGT_TEST_BIN_REV
            ln -s $SRC_TEST_BIN $SRC_TEST_BIN_REV
        fi
    done


fi