run_wild2max_dev.sh

#!/bin/bash

# Get the parent directory of this script
SCRIPT_DIR="$(dirname $(realpath "$0"))"

# List of genres to be evaluated
GENRES=(
    "chatgpt"
    "children"
    "news"
    "novels"
    "poetry"
    "un"
)

DATA_DIR="${SCRIPT_DIR}/data/wild2max/dev"
PRED_DIR="${SCRIPT_DIR}/output/predictions/wild2max/dev"
EVAL_DIR="${SCRIPT_DIR}/output/eval/wild2max"
DB_ORIG_PATH="${SCRIPT_DIR}/databases/calima-msa-s31.db"
DB_EXT_PATH="${SCRIPT_DIR}/databases/calima-msa-s31-extended.db"

# Function that generates diacritization predictions using original CAMeL Tools
# for a given genre.
function gen_predictions_original () {
    GENRE="$1"
    INPUT_PATH="${DATA_DIR}/${GENRE}.tsv"
    OUTPUT_PATH="${PRED_DIR}/${GENRE}.original.tsv"

    printf ">>> Generating diac predictions for dev/${GENRE} using original CAMeL Tools...\n"

    python3 "${SCRIPT_DIR}/gen_predictions_word.py" "${INPUT_PATH}" \
        --db "${DB_ORIG_PATH}" \
        --ct-noctx --oracle-noctx \
        --output "${OUTPUT_PATH}"

    if [ $? -ne 0 ]; then
        exit 1
    fi
}

# Function that generates diacritization predictions using modified CAMeL Tools
# with extentions for a given genre.
function gen_predictions_extended () {
    GENRE="$1"
    INPUT_PATH="${DATA_DIR}/${GENRE}.tsv"
    OUTPUT_PATH="${PRED_DIR}/${GENRE}.extended.tsv"

    printf ">>> Generating diac predictions for dev/${GENRE} using extended CAMeL Tools...\n"

    python3 "${SCRIPT_DIR}/gen_predictions_word.py" "${INPUT_PATH}" \
        --db "${DB_EXT_PATH}" \
        --ct-soloctx --ct-fullctx --ctpp-soloctx --ctpp-fullctx \
        --oracle-soloctx --oracle-fullctx \
        --output ${OUTPUT_PATH}

    if [ $? -ne 0 ]; then
        exit 1
    fi
}

# Function that evaluates predictions generated using original CAMeL Tools for
# a given list of genres.
function evaluate_original () {
    GENRES="$1"
    OUTPUT_PATH="${EVAL_DIR}/wild2max-dev.original.tsv"
    GENRE_PATH_ARGS=()

    for GENRE in ${GENRES[@]}; do
        GENRE_PATH_ARGS+=("${GENRE}:${PRED_DIR}/${GENRE}.original.tsv")
    done

    printf ">>> Evaluating predictions for dev/* using original CAMeL Tools...\n"

    python3 "${SCRIPT_DIR}/evaluate.py" --output "${OUTPUT_PATH}" \
        --ct-noctx --oracle-noctx --oov \
        --norm-diac --norm-consonants \
        ${GENRE_PATH_ARGS[@]}

    if [ $? -ne 0 ]; then
        exit 1
    fi
}

# Function that evaluates predictions generated using modified CAMeL Tools with
# extentions for a given list of genres.
function evaluate_extended () {
    GENRES="$1"
    OUTPUT_PATH="${EVAL_DIR}/wild2max-dev.extended.tsv"
    GENRE_PATH_ARGS=()

    for GENRE in ${GENRES[@]}; do
        GENRE_PATH_ARGS+=("${GENRE}:${PRED_DIR}/${GENRE}.extended.tsv")
    done

    printf ">>> Evaluating predictions for dev/* using extended CAMeL Tools...\n"

    python3 "${SCRIPT_DIR}/evaluate.py" --output "${OUTPUT_PATH}" \
        --ct-soloctx --ct-fullctx --ctpp-soloctx --ctpp-fullctx \
        --oracle-soloctx --oracle-fullctx --oov \
        --norm-diac --norm-consonants \
        ${GENRE_PATH_ARGS[@]}

    if [ $? -ne 0 ]; then
        exit 1
    fi
}

# Use custom CAMeL Tools data directory.
export CAMELTOOLS_DATA="${SCRIPT_DIR}/envs/ct_data"

# Create output directories if they don't already exist.
mkdir -p "${PRED_DIR}"
mkdir -p "${EVAL_DIR}"

# Deactivate the current environment (if any).
deactivate &> /dev/null

# Activate the modified CAMeL Tools environment.
source "${SCRIPT_DIR}/envs/modified/bin/activate"

# Generate predictions using modified CAMeL Tools with extentions.
for GENRE in ${GENRES[@]}; do
    gen_predictions_extended ${GENRE}
done

# Deactivate the modified CAMeL Tools environment.
deactivate

# Activate the original CAMeL Tools environment.
source "${SCRIPT_DIR}/envs/original/bin/activate"

# Generate predictions using original CAMeL Tools.
for GENRE in ${GENRES[@]}; do
    gen_predictions_original ${GENRE}
done

# Evaluate original and extended predictions seperately.
evaluate_original ${GENRES}
evaluate_extended ${GENRES}

# Deactivate the original CAMeL Tools environment.
deactivate