ShearedLlama-bi-mntp-simcse.json

{
    "model_name_or_path": "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp",
    "peft_addr": "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse",
    "model_class": "custom",
    "bidirectional": true,
    "classifier_dropout": 0.1,
    "torch_dtype": "bfloat16",
    "attn_implementation": "flash_attention_2",
    "merge_subwords": true,
    "retroactive_labels": "next_token",
    "output_dir": "output/word-task/pos_tags/ShearedLlama/bi-mntp-simcse",
    "dataset_name": "conll2003",
    "task": "pos_tags",
    "num_train_epochs": 10,
    "learning_rate": 5e-4,
    "do_train": true,
    "save_steps": 250,
    "logging_steps": 50,
    "save_strategy": "steps",
    "do_eval": true,
    "eval_steps": 500,
    "evaluation_strategy": "steps",
    "overwrite_output_dir": true,
    "stop_after_n_steps": 4000,
    "remove_unused_columns": false
}