src/eval_modules/eval-hf_v2.sh

#!/bin/bash

BASEDIR=$(dirname "$0")
cd $BASEDIR/..
echo Current Directory:
pwd

export LLM_MODEL_TYPE=huggingface

export ORG_NAME=$1
export MODEL_NAME=$2
export QA_WITH_MS_MACRO=$3

if [[ $QA_WITH_MS_MACRO == 'true' ]]; then
    export QUESTIONS_FILE_PATH="./data/datasets/ms_macro.json"
    export RESULT_FILENAME_PREFIX_BASE=${MODEL_NAME}_mm
else
    export QUESTIONS_FILE_PATH="./data/datasets/WebQSP.test.wikidata.json"
    export RESULT_FILENAME_PREFIX_BASE=${MODEL_NAME}_wd
fi

echo QA_WITH_MS_MACRO=$QA_WITH_MS_MACRO
echo RESULT_FILENAME_PREFIX_BASE=$RESULT_FILENAME_PREFIX_BASE

export HUGGINGFACE_MODEL_NAME_OR_PATH="$ORG_NAME/$MODEL_NAME"

export QA_WITH_RAG=true
export APPLY_CHAT_TEMPLATE_FOR_RAG=true
export RESULT_FILENAME_PREFIX=${RESULT_FILENAME_PREFIX_BASE}_${APPLY_CHAT_TEMPLATE_FOR_RAG}
echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH with $QUESTIONS_FILE_PATH
echo RESULT_FILENAME_PREFIX=$RESULT_FILENAME_PREFIX
python tune_rp.py

export APPLY_CHAT_TEMPLATE_FOR_RAG=false
export RESULT_FILENAME_PREFIX=${RESULT_FILENAME_PREFIX_BASE}_${APPLY_CHAT_TEMPLATE_FOR_RAG}
echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH with $QUESTIONS_FILE_PATH
echo RESULT_FILENAME_PREFIX=$RESULT_FILENAME_PREFIX
python tune_rp.py

export QA_WITH_RAG=false
export RESULT_FILENAME_PREFIX=${RESULT_FILENAME_PREFIX_BASE}_non_rag
echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH with $QUESTIONS_FILE_PATH
echo RESULT_FILENAME_PREFIX=$RESULT_FILENAME_PREFIX
python tune_rp.py