13-significance-tests.sh

#!/bin/bash
#SBATCH --time=01:00:00
#SBATCH --mem=22G
#SBATCH --job-name=significance_tests
#SBATCH --output=log/%x_%j.out

exp_name=$1
datadir=$2
src_lang=$3
tgt_lang=$4
vocab_size_src_baseline=$5
vocab_size_src_system=$6
vocab_size_tgt=$7
if [ -z "$exp_name" ] || [ -z "$datadir" ] || [ -z "$src_lang" ] || \
    [ -z "$tgt_lang" ] || [ -z "$vocab_size_src_baseline" ] || \
    [ -z "$vocab_size_src_system" ] || [ -z "$vocab_size_tgt" ]; then
    echo "Usage: $0 <exp_name> <datadir> <src_lang> <tgt_lang> \\"
    echo "   <vocab_size_src_baseline> <vocab_size_tgt_baseline> \\"
    echo "   <vocab_size_tgt_system>"
    exit 1
fi
. ./exp/${exp_name}/config.sh

nmt_dir_baseline="${datadir}/nmt-${src_lang}-${tgt_lang}_vocabs_${vocab_size_src_baseline}_${vocab_size_tgt}"
nmt_dir_system="${datadir}/nmt-${src_lang}-${tgt_lang}_vocabs_${vocab_size_src_system}_${vocab_size_tgt}"

model_cp_baseline=$(cat "${nmt_dir_baseline}/best_model_cp.txt")
model_cp_system=$(cat "${nmt_dir_system}/best_model_cp.txt")

ref_target="${datadir}/raw_${tgt_lang}_test_full.txt"
pred_target_baseline="${nmt_dir_baseline}/test_pred_cp${model_cp_baseline}_full.txt"
pred_target_system="${nmt_dir_system}/test_pred_cp${model_cp_system}_full.txt"

echo "exp_name: ${exp_name}"
echo "datadir: ${datadir}"
echo "src_lang: ${src_lang}"
echo "tgt_lang: ${tgt_lang}"
echo "reference target: $ref_target"
echo "predicted target baseline: $pred_target_baseline"
echo "predicted target system: $pred_target_system"

if [ ! -f "${pred_target_baseline%.txt}.detok" ]; then
    (set -x; python utils/postprocess_translations.py \
        "${ref_target}" \
        "${pred_target_baseline}" \
        "${pred_target_baseline%.txt}.detok" \
        --parse-aligned \
        --desubword-hyp) || exit 1
fi
if [ ! -f "${pred_target_system%.txt}.detok" ]; then
    (set -x; python utils/postprocess_translations.py \
        "${ref_target}" \
        "${pred_target_system}" \
        "${pred_target_system%.txt}.detok" \
        --parse-aligned \
        --desubword-hyp) || exit 1
fi

(set -x; sacrebleu \
    "$ref_target" \
    --width 4 \
    --input "${pred_target_baseline%.txt}.detok" \
        "${pred_target_system%.txt}.detok" \
    --metrics bleu chrf --chrf-word-order 2 \
    --paired-bs --paired-bs-n 3000 \
    > "${pred_target_system%.txt}.paired-bs") || exit 1

(set -x; sacrebleu \
    "$ref_target" \
    --width 4 \
    --input "${pred_target_baseline%.txt}.detok" \
        "${pred_target_system%.txt}.detok" \
    --metrics bleu chrf --chrf-word-order 2 \
    --paired-ar \
    > "${pred_target_system%.txt}.paired-ar") || exit 1

if [ -z "$SLURM_JOB_ID" ]; then
    exit 0
fi
if [ -f "log/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.out" ]; then
    sleep 20 # wait for the file to be written
    cp "log/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.out" \
        "${nmt_dir_system}/" || true
fi