meta_train_gpt2.yaml

# @package _global_

defaults:
  - override /default: default.yaml

# model_type: gemma-2b
# model_name_or_path: google/gemma-2b

model_type: gpt2
model_name_or_path: gpt2

# experiment param:
do_train: true
do_eval: false

multi_task: false
baseline: false
use_lora: false
freeze_partial: false
condition: false
peft: false

wandb_name: gpt2-meta-4k-32bit
wandb_project: meta-memory

# checkpoint param:
load_checkpoint: false
checkpoint: None
wandb_model: false
wandb_checkpoint: false

# train param:
train_batch_size: 1
predict_batch_size: 1
eval_batch_size: 1
num_train_epochs: 2
callback_monitor: val_accuracy

# inner param:
n_inner_iter: 8
dyna_lr: true
inner_verbose: false
inner_mode: all

inner_lr: 5e-5                    # the learning rate used for inner loop optimization
inner_grad_accumulate: false      # whether to accumulate gradients in the inner loop
inner_accumulate_steps: 1         # the number of steps to accumulate gradients in the inner loop

# data_param:
dataset: babilong               # student_records, loss_in_mid, babilong
dataset_type: babilong           # student_records, loss_in_mid, babilong
load_order: in
no_facts: false
random_facts: false
no_question: false
max_eval_data: 0

device_idx: 0