Skip to content

Commit

Permalink
fix: Resolves HumanEval Inference by replacing task_id with instance_…
Browse files Browse the repository at this point in the history
…id (#4364)

Co-authored-by: Harshit Surana <surana.h@gmail.com>
  • Loading branch information
Ethan0456 and suranah authored Oct 15, 2024
1 parent 6bbd75c commit 173018e
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions evaluation/humanevalfix/run_infer.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def get_config(


def _get_instance_id(instance: pd.Series) -> str:
return instance.task_id.replace('/', '__')
return instance.instance_id.replace('/', '__')


def initialize_runtime(
Expand Down Expand Up @@ -206,9 +206,9 @@ def process_instance(
# Setup the logger properly, so you can run multi-processing to parallelize the evaluation
if reset_logger:
log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
reset_logger_for_multiprocessing(logger, instance.task_id, log_dir)
reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
else:
logger.info(f'Starting evaluation for instance {instance.task_id}.')
logger.info(f'Starting evaluation for instance {instance.instance_id}.')

# Create file with HumanEvalFix problem
# Prompt reference: https://github.com/bigcode-project/bigcode-evaluation-harness/blob/84b96da31b7f840b55c5733325346176140cdb6b/bigcode_eval/tasks/humanevalpack.py#L509
Expand Down Expand Up @@ -257,7 +257,7 @@ def process_instance(

# Save the output
output = EvalOutput(
instance_id=instance.task_id,
instance_id=instance.instance_id,
instruction=instruction,
metadata=metadata,
history=histories,
Expand Down

0 comments on commit 173018e

Please sign in to comment.