diff --git a/scripts/eval/yamls/coding_tasks.yaml b/scripts/eval/yamls/coding_tasks.yaml index 3a19dc640d..727b9a6078 100644 --- a/scripts/eval/yamls/coding_tasks.yaml +++ b/scripts/eval/yamls/coding_tasks.yaml @@ -6,6 +6,8 @@ icl_tasks: pass_at_k: 1 num_beams: 20 icl_task_type: code_evaluation + batch_size: 1 + - label: human_eval_cpp dataset_uri: eval/local_data/programming/processed_human_eval_cpp.jsonl # ADD YOUR OWN DATASET URI @@ -13,6 +15,7 @@ icl_tasks: pass_at_k: 1 num_beams: 20 icl_task_type: code_evaluation + batch_size: 1 - label: human_eval_js dataset_uri: eval/local_data/programming/processed_human_eval_js.jsonl # ADD YOUR OWN DATASET URI @@ -20,3 +23,4 @@ icl_tasks: pass_at_k: 1 num_beams: 20 icl_task_type: code_evaluation + batch_size: 1 diff --git a/scripts/eval/yamls/tasks.yaml b/scripts/eval/yamls/tasks.yaml index 54d14e34ab..187b15ee88 100644 --- a/scripts/eval/yamls/tasks.yaml +++ b/scripts/eval/yamls/tasks.yaml @@ -180,6 +180,7 @@ icl_tasks: pass_at_k: 1 num_beams: 20 icl_task_type: code_evaluation + batch_size: 1 - label: human_eval_cpp dataset_uri: eval/local_data/programming/processed_human_eval_cpp.jsonl # ADD YOUR OWN DATASET URI @@ -187,6 +188,7 @@ icl_tasks: pass_at_k: 1 num_beams: 20 icl_task_type: code_evaluation + batch_size: 1 - label: human_eval_js dataset_uri: eval/local_data/programming/processed_human_eval_js.jsonl # ADD YOUR OWN DATASET URI @@ -194,3 +196,4 @@ icl_tasks: pass_at_k: 1 num_beams: 20 icl_task_type: code_evaluation + batch_size: 1