diff --git a/e2eAIOK/deltatuner/README.md b/e2eAIOK/deltatuner/README.md index b6d6d5b40..3c503e1b8 100644 --- a/e2eAIOK/deltatuner/README.md +++ b/e2eAIOK/deltatuner/README.md @@ -84,10 +84,10 @@ Please refer to [example page](https://github.com/intel/e2eAIOK/tree/main/exampl | Model | LoRA | SSF | |--------------| ---- | ---- | | GPT-2 | ✅ | | -| GPT-J | ✅ | | -| Bloom | ✅ | | -| OPT | ✅ | | -| GPT-Neo | ✅ | | -| Falcon | ✅ | | +| GPT-J | ✅ | ✅ | +| Bloom | ✅ | ✅ | +| OPT | ✅ | ✅ | +| GPT-Neo | ✅ | ✅ | +| Falcon | ✅ | ✅ | | LLaMA | ✅ | ✅ | | MPT | ✅ | ✅ | diff --git a/e2eAIOK/deltatuner/deltatuner/deltatuner.py b/e2eAIOK/deltatuner/deltatuner/deltatuner.py index f6a3adbf3..2c172d98b 100644 --- a/e2eAIOK/deltatuner/deltatuner/deltatuner.py +++ b/e2eAIOK/deltatuner/deltatuner/deltatuner.py @@ -39,9 +39,9 @@ def optimize(model, tokenizer, algo: str="auto", adapter_name: str="default", de if isinstance(peft_config, LoraConfig) or isinstance(peft_config, SSFConfig): if peft_config.task_type not in MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING: - model = DeltaTunerModel(model, tokenizer, peft_config, adapter_name, deltatuning_args) + model = DeltaTunerModel(model, peft_config, adapter_name, deltatuning_args, tokenizer) else: - model = MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING[peft_config.task_type](model, tokenizer, peft_config, adapter_name, deltatuning_args) + model = MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING[peft_config.task_type](model, peft_config, adapter_name, deltatuning_args, tokenizer) return model else: raise NotImplementedError("Current algorithm {} is not supported in deltatuner. ".format(algo)) \ No newline at end of file diff --git a/e2eAIOK/deltatuner/deltatuner/deltatuner_model.py b/e2eAIOK/deltatuner/deltatuner/deltatuner_model.py index 8d6baefaa..917c55f18 100644 --- a/e2eAIOK/deltatuner/deltatuner/deltatuner_model.py +++ b/e2eAIOK/deltatuner/deltatuner/deltatuner_model.py @@ -52,7 +52,7 @@ def setup_seed(seed): torch.manual_seed(seed) class DeltaTunerModel(PeftModel, torch.nn.Module): - def __init__(self, model, tokenizer: AutoTokenizer, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None): + def __init__(self, model, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None, tokenizer: AutoTokenizer = None): torch.nn.Module.__init__(self) self.base_model = model self.tokenizer = tokenizer @@ -104,7 +104,8 @@ def _init_denas_params_(self): self.denas_config.model_id = self.base_model.config._name_or_path self.denas_config.tokenizer = self.tokenizer self.denas_config.max_param_limits = sum(param.numel() for param in self.base_model.parameters() if param.requires_grad) / 10.**6 if self.denas_config.max_param_limits is None else self.denas_config.max_param_limits - self.denas_config.budget_latency_max = network_latency(self.base_model, self.tokenizer, batch_size=self.denas_config.batch_size) if self.denas_config.budget_latency_max is not None else self.denas_config.budget_latency_max + if self.tokenizer: + self.denas_config.budget_latency_max = network_latency(self.base_model, self.tokenizer, batch_size=self.denas_config.batch_size) if self.denas_config.budget_latency_max is not None else self.denas_config.budget_latency_max def search(self, denas_config, super_net, search_space): setup_seed(denas_config.random_seed) @@ -219,6 +220,13 @@ def from_pretrained( from .mapping import DELTATUNER_TYPE_TO_CONFIG_MAPPING, MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING denas_config = kwargs.pop("denas_config", None) + best_structure_file = os.path.join(model_id, "best_model_structure.txt") + if os.path.isfile(best_structure_file): + denas_config.denas = True + denas_config.best_model_structure = best_structure_file + else: + denas_config.denas = False + # load the config if config is None: peft_type_name = PeftConfig._get_peft_type( @@ -376,8 +384,8 @@ def load_adapter(self, model_id: str, adapter_name: str, is_trainable: bool = Fa return load_result class DelatunerModelForCausalLM(DeltaTunerModel): - def __init__(self, model: PeftModel, tokenizer: AutoTokenizer, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None): - super().__init__(model, tokenizer, peft_config, adapter_name, denas_config) + def __init__(self, model: PeftModel, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None, tokenizer: AutoTokenizer = None): + super().__init__(model, peft_config, adapter_name, denas_config, tokenizer) self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation def forward( diff --git a/e2eAIOK/deltatuner/deltatuner/tuner/deltatuner_ssf.py b/e2eAIOK/deltatuner/deltatuner/tuner/deltatuner_ssf.py index 48cff341b..2a8716695 100644 --- a/e2eAIOK/deltatuner/deltatuner/tuner/deltatuner_ssf.py +++ b/e2eAIOK/deltatuner/deltatuner/tuner/deltatuner_ssf.py @@ -1,3 +1,4 @@ +import os import math import re import warnings @@ -80,6 +81,50 @@ class SSFConfig(PeftConfig): def __post_init__(self): self.peft_type = DeltaTunerType.SSF + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: str, subfolder: Optional[str] = None, **kwargs): + r""" + This method loads the configuration of your adapter model from a directory. + + Args: + pretrained_model_name_or_path (`str`): + The directory or the Hub repository id where the configuration is saved. + kwargs (additional keyword arguments, *optional*): + Additional keyword arguments passed along to the child class initialization. + """ + from peft.utils import CONFIG_NAME + + path = ( + os.path.join(pretrained_model_name_or_path, subfolder) + if subfolder is not None + else pretrained_model_name_or_path + ) + + hf_hub_download_kwargs, class_kwargs, _ = cls._split_kwargs(kwargs) + + if os.path.isfile(os.path.join(path, CONFIG_NAME)): + config_file = os.path.join(path, CONFIG_NAME) + else: + try: + config_file = hf_hub_download( + pretrained_model_name_or_path, CONFIG_NAME, subfolder=subfolder, **hf_hub_download_kwargs + ) + except Exception: + raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'") + + loaded_attributes = cls.from_json_file(config_file) + + config_cls = cls + + config = config_cls(**class_kwargs) + + for key, value in loaded_attributes.items(): + if hasattr(config, key): + setattr(config, key, value) + + return config + + class DeltaSSFSearchSpace: @classmethod def generate_search_space(cls, model, denas_config): diff --git a/e2eAIOK/deltatuner/deltatuner/utils/config.py b/e2eAIOK/deltatuner/deltatuner/utils/config.py index 1b3fea166..64d34c058 100644 --- a/e2eAIOK/deltatuner/deltatuner/utils/config.py +++ b/e2eAIOK/deltatuner/deltatuner/utils/config.py @@ -1,11 +1,13 @@ import enum from dataclasses import field from typing import List, Optional, Tuple, Union +from peft.utils import TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING class DeltaTunerType(str, enum.Enum): SSF = "SSF" -TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING = { +TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING +TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING.update({ "llama": ["q_proj", "v_proj"], "mpt": ["Wqkv","out_proj","up_proj","down_proj"] -} \ No newline at end of file +}) \ No newline at end of file diff --git a/example/instruction_tuning_pipeline/README.md b/example/instruction_tuning_pipeline/README.md index 59b047b25..6c2779a0e 100644 --- a/example/instruction_tuning_pipeline/README.md +++ b/example/instruction_tuning_pipeline/README.md @@ -125,6 +125,10 @@ python example/instruction_tuning_pipeline/finetune_clm.py \ --bf16 True 2>&1 | tee log/mpt-lora-run-1epoch.log ``` +- If you are using 4th Xeon or later (SPR etc.), please specify the `--bf16 --no_cuda` args; +- If you are using 3th Xeon or before (ICX etc.): please specify the `--no_cuda` args; +- If you are using GPU server: please specify the `--fp16` args. + ## Evaluate the model For model evaluation, we follow the same method in [open_llm_leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard), which evaluate 4 key benchmarks in the [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness). diff --git a/example/instruction_tuning_pipeline/finetune_clm.py b/example/instruction_tuning_pipeline/finetune_clm.py index 6da81f8fc..6740ad098 100755 --- a/example/instruction_tuning_pipeline/finetune_clm.py +++ b/example/instruction_tuning_pipeline/finetune_clm.py @@ -20,6 +20,8 @@ import datasets import logging import os +import json +import errno import sys import transformers from transformers.modeling_utils import unwrap_model @@ -292,6 +294,10 @@ class FinetuneArguments: default=False, metadata={"help": "save merged model"}, ) + merge_model_code_dir: Optional[str] = field( + default="", + metadata={"help": "the code path of base model with enable bias on target modules for ssf algo"}, + ) PROMPT_DICT = { "prompt_with_input": ( @@ -700,12 +706,7 @@ def concatenate_data(dataset, max_seq_length): if finetune_args.delta: if finetune_args.resume_peft != "": - if finetune_args.delta == "ssf": - peft_config = SSFConfig(target_modules=deltatuner_args.ssf_target_modules, - bias="none", - task_type="CAUSAL_LM", - ) - model = DeltaTunerModel.from_pretrained(model, finetune_args.resume_peft, config=peft_config, denas_config=deltatuner_args) + model = DeltaTunerModel.from_pretrained(model, finetune_args.resume_peft, denas_config=deltatuner_args) else: model = deltatuner.optimize(model, tokenizer, algo=finetune_args.delta, deltatuning_args=deltatuner_args) logger.info("***deltatuner optimized model parameter***") @@ -757,6 +758,37 @@ def concatenate_data(dataset, max_seq_length): training_args.output_dir, state_dict=unwrapped_model.state_dict() ) + if finetune_args.save_merged_model: + if isinstance(model, PeftModel): + model = model.merge_and_unload() + saved_dir = os.path.join(training_args.output_dir, "merged_model") + os.makedirs(saved_dir, exist_ok=True) + print(f"copy base model config to {saved_dir}") + os.system(f"cp {model_args.model_name_or_path}/* {saved_dir}") + print(f"remove unnecessary file from {model_args.model_name_or_path}") + os.system(f"rm {saved_dir}/*.bin* {saved_dir}/*.safetensors*") + print(f"Save merged model to {saved_dir}") + torch.save(model.state_dict(), os.path.join(saved_dir, "pytorch_model.bin")) + if finetune_args.delta == 'ssf': + if os.path.exists(finetune_args.merge_model_code_dir): + print(f"copy merged model code to {saved_dir}") + os.system(f"cp {finetune_args.merge_model_code_dir}/* {saved_dir}") + else: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), finetune_args.merge_model_code_dir) + pre_train_config_file = os.path.join(saved_dir, "config.json") + with open(pre_train_config_file, "r") as file: + config_json = json.load(file) + adapter_config_file = os.path.join(finetune_args.resume_peft, "adapter_config.json") + with open(adapter_config_file, "r") as file: + adapter_config_json = json.load(file) + config_json['target_modules'] = adapter_config_json['target_modules'] + + best_structure_file = os.path.join(finetune_args.resume_peft, "best_model_structure.txt") + if os.path.isfile(best_structure_file): + with open(best_structure_file, "r") as file: + best_structure_json = json.loads(file.readline().strip()) + config_json['best_model_structure'] = best_structure_json + # Evaluation if training_args.do_eval: logger.info("*** Evaluate ***") @@ -771,12 +803,6 @@ def concatenate_data(dataset, max_seq_length): trainer.log_metrics("eval", metrics) trainer.save_metrics("eval", metrics) - if finetune_args.save_merged_model: - if isinstance(model, PeftModel): - merged_model = model.merge_and_unload() - os.makedirs(os.path.join(training_args.output_dir,"merged_model"),exist_ok=True) - torch.save(merged_model.state_dict(),os.path.join(training_args.output_dir,"merged_model","pytorch_model.bin")) - print(f"Save merged model to {training_args.output_dir}") def profile_model(model, train_dataset, data_collator, args): from transformers import get_scheduler diff --git a/tests/deltatuner/cicd/run_denas_lora.sh b/tests/deltatuner/cicd/run_denas_lora.sh index 6b165109c..35226069d 100644 --- a/tests/deltatuner/cicd/run_denas_lora.sh +++ b/tests/deltatuner/cicd/run_denas_lora.sh @@ -7,56 +7,41 @@ pip uninstall wandb -y cd /home/vmagent/app/e2eaiok -mkdir -p log models - DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" -# fine-tune mpt-7b with denas-lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path $DATA_PATH"/mpt-7b" \ - --train_file $DATA_PATH"/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/mpt_denas-lora_model \ - --peft lora \ - --delta lora \ - --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \ - --trust_remote_code True \ - --no_cuda \ - 2>&1 | tee log/mpt-denas-lora-run-1epoch.log +mkdir -p $LOG_PATH $MODEL_SAVE_PATH -# fine-tune llama2-7b with denas-lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path $DATA_PATH"/Llama-2-7b-hf" \ - --train_file $DATA_PATH"/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/llama2_denas-lora_model \ - --peft lora \ - --delta lora \ - --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \ - --trust_remote_code True \ - --no_cuda \ - 2>&1 | tee log/llama2-denas-lora-run-1epoch.log +# fine-tune with denas-lora +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_denas-lora" + log_save_path=$LOG_PATH"/"${model_name}"_denas-lora-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file $DATA_PATH"/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --do_train \ + --do_eval \ + --validation_split_percentage 30 \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --output_dir $model_save_path \ + --peft lora \ + --delta lora \ + --denas True \ + --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \ + --trust_remote_code True \ + --no_cuda \ + 2>&1 | tee $log_save_path +done diff --git a/tests/deltatuner/cicd/run_denas_ssf.sh b/tests/deltatuner/cicd/run_denas_ssf.sh index 07c4aef89..e62794cc2 100644 --- a/tests/deltatuner/cicd/run_denas_ssf.sh +++ b/tests/deltatuner/cicd/run_denas_ssf.sh @@ -6,38 +6,21 @@ pip uninstall wandb -y cd /home/vmagent/app/e2eaiok -mkdir -p log - DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" -#run mpt with ssf and denas, bf16 -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16" \ - --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \ - --delta ssf \ - --denas True \ - 2>&1 | tee "log/mpt-7b-ssf-allmodules-denas-bf16-run-1epoch.log" +mkdir -p $LOG_PATH $MODEL_SAVE_PATH -#run llama with ssf and denas, bf16 -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ +# fine-tune with denas-ssf +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_denas-ssf" + log_save_path=$LOG_PATH"/"${model_name}"_denas-ssf-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ --train_file "$DATA_PATH/alpaca_data.json" \ --dataset_concatenation \ --per_device_train_batch_size 8 \ @@ -54,8 +37,9 @@ python example/instruction_tuning_pipeline/finetune_clm.py \ --save_strategy epoch \ --trust_remote_code True \ --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16" \ + --output_dir $model_save_path \ --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \ --delta ssf \ --denas True \ - 2>&1 | tee log/llama2-7b-ssf-denas-bf16-1epoch.log \ No newline at end of file + 2>&1 | tee $log_save_path +done diff --git a/tests/deltatuner/cicd/run_lora.sh b/tests/deltatuner/cicd/run_lora.sh index 8db7a6f86..923463ecb 100644 --- a/tests/deltatuner/cicd/run_lora.sh +++ b/tests/deltatuner/cicd/run_lora.sh @@ -7,56 +7,41 @@ pip uninstall wandb -y cd /home/vmagent/app/e2eaiok -mkdir -p log models - DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" + +mkdir -p $LOG_PATH $MODEL_SAVE_PATH -# fine-tune mpt-7b with denas-lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path $DATA_PATH"/mpt-7b" \ - --train_file $DATA_PATH"/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/mpt_lora_model \ - --peft lora \ - --denas False \ - --debugs \ - --trust_remote_code True \ - --no_cuda \ - 2>&1 | tee log/mpt-lora-run-1epoch.log -# fine-tune llama2-7b with denas-lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path $DATA_PATH"/Llama-2-7b-hf" \ - --train_file $DATA_PATH"/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/llama2_lora_model \ - --peft lora \ - --denas False \ - --debugs \ - --trust_remote_code True \ - --no_cuda \ - 2>&1 | tee log/llama2-lora-run-1epoch.log +# fine-tune with lora +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"-lora" + log_save_path=$LOG_PATH"/"${model_name}"-lora-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file $DATA_PATH"/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --do_train \ + --do_eval \ + --validation_split_percentage 30 \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --output_dir $model_save_path \ + --peft lora \ + --denas False \ + --debugs \ + --trust_remote_code True \ + --no_cuda \ + 2>&1 | tee $log_save_path +done diff --git a/tests/deltatuner/cicd/run_ssf.sh b/tests/deltatuner/cicd/run_ssf.sh index 4bb1800e9..f1c3cbc0d 100644 --- a/tests/deltatuner/cicd/run_ssf.sh +++ b/tests/deltatuner/cicd/run_ssf.sh @@ -6,38 +6,21 @@ pip uninstall wandb -y cd /home/vmagent/app/e2eaiok -mkdir -p log - DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" -#run mpt with ssf, bf16 -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-bf16" \ - --delta ssf \ - --denas False \ - --debugs \ - 2>&1 | tee log/mpt-ssf-run-allmodules-bf16-1epoch.log - -#run llama2 with ssf, bf16 -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ +mkdir -p $LOG_PATH $MODEL_SAVE_PATH + +# fine-tune with ssf +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_ssf" + log_save_path=$LOG_PATH"/"${model_name}"_ssf-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ --train_file "$DATA_PATH/alpaca_data.json" \ --dataset_concatenation \ --per_device_train_batch_size 8 \ @@ -54,8 +37,9 @@ python example/instruction_tuning_pipeline/finetune_clm.py \ --save_strategy epoch \ --trust_remote_code True \ --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-bf16" \ + --output_dir $model_save_path \ --delta ssf \ --denas False \ --debugs \ - 2>&1 | tee log/llama2-7b-ssf-bf16-1epoch.log \ No newline at end of file + 2>&1 | tee $log_save_path +done diff --git a/tests/deltatuner/cicd/run_ssf_model.sh b/tests/deltatuner/cicd/run_ssf_model.sh new file mode 100644 index 000000000..a4e954abd --- /dev/null +++ b/tests/deltatuner/cicd/run_ssf_model.sh @@ -0,0 +1,47 @@ +set -x + +cd /home/vmagent/app/e2eaiok/e2eAIOK/deltatuner +pip install -e . + +cd /home/vmagent/app/e2eaiok + +DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" + +mkdir -p $LOG_PATH $MODEL_SAVE_PATH + +# fine-tune with ssf +# mosaicml/mpt-7b gpt2 EleutherAI/gpt-j-6b bigscience/bloom-560m facebook/opt-125m EleutherAI/gpt-neo-125m tiiuae/falcon-7b +model_name_list="EleutherAI/gpt-j-6b bigscience/bloom-560m facebook/opt-125m EleutherAI/gpt-neo-125m tiiuae/falcon-7b" +for model_name in $model_name_list +do + model_name_or_path=${model_name} + short_model_name=`echo $model_name | cut -d/ -f2` + model_save_path=${MODEL_SAVE_PATH}"/"${short_model_name}"_ssf" + log_save_path=$LOG_PATH"/"${short_model_name}"_ssf-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file "$DATA_PATH/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --validation_split_percentage 30 \ + --do_train \ + --do_eval \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --trust_remote_code True \ + --output_dir $model_save_path \ + --delta ssf \ + --denas False \ + --debugs \ + --no_cuda \ + 2>&1 | tee $log_save_path + # rm -rf ~/.cache/huggingface/hub +done diff --git a/tests/deltatuner/finetune/run_denas_lora.sh b/tests/deltatuner/finetune/run_denas_lora.sh index 801f4e6da..0904bb613 100644 --- a/tests/deltatuner/finetune/run_denas_lora.sh +++ b/tests/deltatuner/finetune/run_denas_lora.sh @@ -1,56 +1,48 @@ #!/bin/bash set -x -mkdir -p log models -# fine-tune mpt-7b with denas-lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "/home/vmagent/app/dataset/mpt-7b" \ - --train_file "/home/vmagent/app/dataset/stanford_alpaca/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/mpt_denas-lora_model \ - --peft lora \ - --delta lora \ - --trust_remote_code True \ - --no_cuda \ - --bf16 True 2>&1 | tee log/mpt-denas-lora-run-1epoch.log +DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" -# fine-tune llama2-7b with denas-lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "/home/vmagent/app/dataset/Llama-2-7b-hf" \ - --train_file "/home/vmagent/app/dataset/stanford_alpaca/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/llama2_denas-lora_model \ - --peft lora \ - --delta lora \ - --trust_remote_code True \ - --no_cuda \ - --bf16 True 2>&1 | tee log/llama2-denas-lora-run-1epoch.log +mkdir -p $LOG_PATH $MODEL_SAVE_PATH +# fine-tune with denas-lora +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_denas-lora" + log_save_path=$LOG_PATH"/"${model_name}"_denas-lora-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file $DATA_PATH"/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --do_train \ + --do_eval \ + --validation_split_percentage 30 \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --output_dir $model_save_path \ + --peft lora \ + --delta lora \ + --denas True \ + --trust_remote_code True \ + --no_cuda \ + 2>&1 | tee $log_save_path +done -# used for assign target modules to lora -# --lora_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ -# --lora_target_modules Wqkv out_proj up_proj down_proj \ \ No newline at end of file +########################## advanced options ########################## +# note 0: used for assign target modules + # llama: --lora_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ + # mpt: --lora_target_modules Wqkv out_proj up_proj down_proj \ +# note 1: used for 4th Xeon or later (SPR etc.): --bf16 --no_cuda +# note 2: used for 3th Xeon or before (ICX etc.): --no_cuda +# note 3: used for GPU server: --fp16 \ No newline at end of file diff --git a/tests/deltatuner/finetune/run_denas_ssf.sh b/tests/deltatuner/finetune/run_denas_ssf.sh new file mode 100644 index 000000000..e1b68ab04 --- /dev/null +++ b/tests/deltatuner/finetune/run_denas_ssf.sh @@ -0,0 +1,47 @@ +set -x + +DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" + +mkdir -p $LOG_PATH $MODEL_SAVE_PATH + +# fine-tune with denas-ssf +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_denas-ssf" + log_save_path=$LOG_PATH"/"${model_name}"_denas-ssf-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file "$DATA_PATH/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --validation_split_percentage 30 \ + --do_train \ + --do_eval \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --trust_remote_code True \ + --no_cuda \ + --output_dir $model_save_path \ + --delta ssf \ + --denas True \ + 2>&1 | tee $log_save_path +done + + +########################## advanced options ########################## +# note 0: used for assign target modules + # llama: --ssf_target_module q_proj v_proj k_proj o_proj up_proj down_proj \ + # mpt: --ssf_target_module Wqkv out_proj up_proj down_proj \ +# note 1: used for 4th Xeon or later (SPR etc.): --bf16 --no_cuda +# note 2: used for 3th Xeon or before (ICX etc.): --no_cuda +# note 3: used for GPU server: --fp16 \ No newline at end of file diff --git a/tests/deltatuner/finetune/run_lora.sh b/tests/deltatuner/finetune/run_lora.sh index ba4b331ae..91a8bf5b4 100644 --- a/tests/deltatuner/finetune/run_lora.sh +++ b/tests/deltatuner/finetune/run_lora.sh @@ -1,56 +1,49 @@ #!/bin/bash set -x -mkdir -p log models -# fine-tune mpt-7b with lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "/home/vmagent/app/dataset/mpt-7b" \ - --train_file "/home/vmagent/app/dataset/stanford_alpaca/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/mpt_lora_model \ - --peft lora \ - --denas False \ - --trust_remote_code True \ - --no_cuda \ - --bf16 True 2>&1 | tee log/mpt-lora-run-1epoch.log +DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" -# fine-tune llama2-7b with lora -python example/instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "/home/vmagent/app/dataset/Llama-2-7b-hf" \ - --train_file "/home/vmagent/app/dataset/stanford_alpaca/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --do_train \ - --do_eval \ - --validation_split_percentage 30 \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --output_dir models/llama2_lora_model \ - --peft lora \ - --denas False \ - --trust_remote_code True \ - --no_cuda \ - --bf16 True 2>&1 | tee log/llama2-lora-run-1epoch.log +mkdir -p $LOG_PATH $MODEL_SAVE_PATH -# used for assign target modules to lora -# --lora_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ -# --lora_target_modules Wqkv out_proj up_proj down_proj \ \ No newline at end of file +# fine-tune with lora +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"-lora" + log_save_path=$LOG_PATH"/"${model_name}"-lora-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file $DATA_PATH"/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --do_train \ + --do_eval \ + --validation_split_percentage 30 \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --output_dir $model_save_path \ + --peft lora \ + --denas False \ + --trust_remote_code True \ + --no_cuda \ + 2>&1 | tee $log_save_path +done + + +########################## advanced options ########################## +# note 0: used for assign target modules + # llama: --lora_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ + # mpt: --lora_target_modules Wqkv out_proj up_proj down_proj \ +# note 1: used for 4th Xeon or later (SPR etc.): --bf16 --no_cuda +# note 2: used for 3th Xeon or before (ICX etc.): --no_cuda +# note 3: used for GPU server: --fp16 diff --git a/tests/deltatuner/finetune/run_ssf.sh b/tests/deltatuner/finetune/run_ssf.sh new file mode 100644 index 000000000..c9732c0ff --- /dev/null +++ b/tests/deltatuner/finetune/run_ssf.sh @@ -0,0 +1,47 @@ +set -x + +DATA_PATH="/home/vmagent/app/data" +LOG_PATH=$DATA_PATH"/dtuner_test/log" +MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models" + +mkdir -p $LOG_PATH $MODEL_SAVE_PATH + +# fine-tune with ssf +model_name_list="mpt-7b Llama-2-7b-hf" +for model_name in $model_name_list +do + model_name_or_path=${DATA_PATH}"/"${model_name} + model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_ssf" + log_save_path=$LOG_PATH"/"${model_name}"_ssf-1epoch.log" + python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $model_name_or_path \ + --train_file "$DATA_PATH/alpaca_data.json" \ + --dataset_concatenation \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --gradient_accumulation_steps 1 \ + --validation_split_percentage 30 \ + --do_train \ + --do_eval \ + --learning_rate 1e-4 \ + --num_train_epochs 1 \ + --logging_steps 100 \ + --save_total_limit 1 \ + --log_level info \ + --save_strategy epoch \ + --trust_remote_code True \ + --no_cuda \ + --output_dir $model_save_path \ + --delta ssf \ + --denas False \ + 2>&1 | tee $log_save_path +done + + +########################## advanced options ########################## +# note 0: used for assign target modules + # llama: --ssf_target_module q_proj v_proj k_proj o_proj up_proj down_proj \ + # mpt: --ssf_target_module Wqkv out_proj up_proj down_proj \ +# note 1: used for 4th Xeon or later (SPR etc.): --bf16 --no_cuda +# note 2: used for 3th Xeon or before (ICX etc.): --no_cuda +# note 3: used for GPU server: --fp16 \ No newline at end of file diff --git a/tests/deltatuner/finetune/ssf/llama2-7b-lora.sh b/tests/deltatuner/finetune/ssf/llama2-7b-lora.sh deleted file mode 100755 index 662074530..000000000 --- a/tests/deltatuner/finetune/ssf/llama2-7b-lora.sh +++ /dev/null @@ -1,24 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-lora" \ - --peft lora \ - --denas False \ - | tee llama2-7b-lora-1epoch.log diff --git a/tests/deltatuner/finetune/ssf/llama2-7b-ssf-allmodules.sh b/tests/deltatuner/finetune/ssf/llama2-7b-ssf-allmodules.sh deleted file mode 100755 index 7077fda90..000000000 --- a/tests/deltatuner/finetune/ssf/llama2-7b-ssf-allmodules.sh +++ /dev/null @@ -1,135 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-allmodules" \ - --ssf_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ - --delta ssf \ - --denas False \ - | tee llama2-7b-ssf-allmodules-1epoch.log - -#bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-allmodules-bf16" \ - --ssf_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ - --delta ssf \ - --bf16 True \ - --denas False \ - | tee llama2-7b-ssf-allmodules-bf16-1epoch.log - -#merge -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-allmodules-merge" \ - --ssf_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ - --delta ssf \ - --resume_peft "$DATA_PATH/llama2-7b-ssf-allmodules" \ - --save_merged_model True - -#merge, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-allmodules-bf16-merge" \ - --ssf_target_modules q_proj v_proj k_proj o_proj up_proj down_proj \ - --delta ssf \ - --resume_peft "$DATA_PATH/llama2-7b-ssf-allmodules-bf16" \ - --save_merged_model True \ - --bf16 True - -#evaluate merged model -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/llama2-7b-ssf-allmodules-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-allmodules-merge/eval_merged" \ - --debugs - -#evaluate merged model, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/llama2-7b-ssf-allmodules-bf16-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-allmodules-bf16-merge/eval_merged" \ - --bf16 True \ - --debugs \ No newline at end of file diff --git a/tests/deltatuner/finetune/ssf/llama2-7b-ssf-denas.sh b/tests/deltatuner/finetune/ssf/llama2-7b-ssf-denas.sh deleted file mode 100755 index aa6fb3704..000000000 --- a/tests/deltatuner/finetune/ssf/llama2-7b-ssf-denas.sh +++ /dev/null @@ -1,157 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas" \ - --delta ssf \ - --denas True \ - | tee llama2-7b-ssf-denas-1epoch.log - -#bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16" \ - --delta ssf \ - --denas True \ - --bf16 True \ - | tee llama2-7b-ssf-denas-bf16-1epoch.log - -#evaluate model, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16/eval" \ - --delta ssf \ - --bf16 True \ - --resume_peft "$DATA_PATH/llama2-7b-ssf-denas-bf16" \ - --denas "$DATA_PATH/llama2-7b-ssf-denas-bf16/best_model_structure.txt" \ - --debugs - -#merge -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-merge" \ - --delta ssf \ - --resume_peft "$DATA_PATH/llama2-7b-ssf-denas" \ - --save_merged_model True \ - --denas "$DATA_PATH/llama2-7b-ssf-denas/best_model_structure.txt" \ - --debugs - -#merge, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16-merge" \ - --delta ssf \ - --bf16 True \ - --resume_peft "$DATA_PATH/llama2-7b-ssf-denas-bf16" \ - --save_merged_model True \ - --denas "$DATA_PATH/llama2-7b-ssf-denas-bf16/best_model_structure.txt" \ - --debugs - -#evaluate merged model -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/llama2-7b-ssf-denas-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-merge/eval_merge" \ - --debugs - -#evaluate merged model, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/llama2-7b-ssf-denas-bf16-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16-merge/eval_merge" \ - --bf16 True \ - --debugs \ No newline at end of file diff --git a/tests/deltatuner/finetune/ssf/mpt-7b-full.sh b/tests/deltatuner/finetune/ssf/mpt-7b-full.sh deleted file mode 100755 index 27211fc0d..000000000 --- a/tests/deltatuner/finetune/ssf/mpt-7b-full.sh +++ /dev/null @@ -1,23 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-full" \ - | tee mpt-lora-run-1epoch.log diff --git a/tests/deltatuner/finetune/ssf/mpt-7b-lora.sh b/tests/deltatuner/finetune/ssf/mpt-7b-lora.sh deleted file mode 100755 index 8b9e695f3..000000000 --- a/tests/deltatuner/finetune/ssf/mpt-7b-lora.sh +++ /dev/null @@ -1,24 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-lora" \ - --peft lora \ - | tee mpt-lora-run-1epoch.log diff --git a/tests/deltatuner/finetune/ssf/mpt-7b-ssf-allmodules-denas.sh b/tests/deltatuner/finetune/ssf/mpt-7b-ssf-allmodules-denas.sh deleted file mode 100755 index 251152050..000000000 --- a/tests/deltatuner/finetune/ssf/mpt-7b-ssf-allmodules-denas.sh +++ /dev/null @@ -1,120 +0,0 @@ -DATA_PATH="/home/data" - -#bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16" \ - --delta ssf \ - --ssf_target_modules Wqkv out_proj up_proj down_proj \ - --bf16 True \ - --denas True \ - | tee "mpt-7b-ssf-allmodules-denas-bf16-run-1epoch.log" - -#merge -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16-merge" \ - --delta ssf \ - --ssf_target_modules Wqkv out_proj up_proj down_proj \ - --resume_peft "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16" \ - --save_merged_model True \ - --denas "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16/best_model_structure.txt" \ - --debugs - -#merge,bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16-merge" \ - --delta ssf \ - --ssf_target_modules Wqkv out_proj up_proj down_proj \ - --bf16 True \ - --resume_peft "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16" \ - --save_merged_model True \ - --denas "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16/best_model_structure.txt" \ - --debugs - -#evaluate merged model -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16-merge/eval_merge" \ - --debugs - -#evaluate merged model, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16-merge/eval_merge" \ - --bf16 True \ - --debugs \ No newline at end of file diff --git a/tests/deltatuner/finetune/ssf/mpt-7b-ssf-allmodules.sh b/tests/deltatuner/finetune/ssf/mpt-7b-ssf-allmodules.sh deleted file mode 100755 index ff9f34385..000000000 --- a/tests/deltatuner/finetune/ssf/mpt-7b-ssf-allmodules.sh +++ /dev/null @@ -1,98 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules" \ - --delta ssf \ - --denas False \ - --ssf_target_modules Wqkv out_proj up_proj down_proj \ - | tee mpt-ssf-run-allmodules-1epoch.log - -# bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-bf16" \ - --delta ssf \ - --denas False \ - --ssf_target_modules Wqkv out_proj up_proj down_proj \ - --bf16 True \ - | tee mpt-ssf-run-allmodules-bf16-1epoch.log - -# merge, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-bf16-merge" \ - --delta ssf \ - --ssf_target_modules Wqkv out_proj up_proj down_proj \ - --bf16 True \ - --resume_peft "$DATA_PATH/mpt-7b-ssf-allmodules-bf16" \ - --save_merged_model True - -#evaluate merged model, bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b-ssf-allmodules-bf16-merge/merged_model" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_eval \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-bf16-merge/eval_merged" \ - --bf16 True \ - --debugs diff --git a/tests/deltatuner/finetune/ssf/mpt-7b-ssf.sh b/tests/deltatuner/finetune/ssf/mpt-7b-ssf.sh deleted file mode 100755 index f4d301b45..000000000 --- a/tests/deltatuner/finetune/ssf/mpt-7b-ssf.sh +++ /dev/null @@ -1,51 +0,0 @@ -DATA_PATH="/home/data" - -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf" \ - --denas False \ - --delta ssf \ - | tee mpt-ssf-run-1epoch.log - -# bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/mpt-7b" \ - --train_file "$DATA_PATH/alpaca_data.json" \ - --dataset_concatenation \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --validation_split_percentage 30 \ - --do_train \ - --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ - --save_total_limit 1 \ - --log_level info \ - --save_strategy epoch \ - --trust_remote_code True \ - --tokenizer_name "$DATA_PATH/gpt-neox-20b" \ - --no_cuda \ - --output_dir "$DATA_PATH/mpt-7b-ssf-bf16" \ - --delta ssf \ - --bf16 True \ - --denas False \ - | tee mpt-ssf-run-1epoch.log \ No newline at end of file diff --git a/tests/deltatuner/finetune/ssf/readme.md b/tests/deltatuner/merge_model/readme.md similarity index 81% rename from tests/deltatuner/finetune/ssf/readme.md rename to tests/deltatuner/merge_model/readme.md index 25bcbd9c4..517c8a45e 100644 --- a/tests/deltatuner/finetune/ssf/readme.md +++ b/tests/deltatuner/merge_model/readme.md @@ -1,6 +1,5 @@ -# Run Deltatuner with ssf +## Run Deltatuner with Merged model -## Llama2-7b ### 1. Finetune the model If use Denas, can use command: @@ -28,7 +27,6 @@ python instruction_tuning_pipeline/finetune_clm.py \ --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16" \ --delta ssf \ --denas True \ - --bf16 True \ | tee llama2-7b-ssf-denas-bf16-1epoch.log ``` @@ -40,13 +38,10 @@ python instruction_tuning_pipeline/finetune_clm.py \ --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ --train_file "$DATA_PATH/alpaca_data.json" \ --dataset_concatenation \ - --per_device_train_batch_size 8 \ --per_device_eval_batch_size 8 \ --gradient_accumulation_steps 1 \ --validation_split_percentage 30 \ --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ --logging_steps 100 \ --save_total_limit 1 \ --log_level info \ @@ -55,22 +50,18 @@ python instruction_tuning_pipeline/finetune_clm.py \ --no_cuda \ --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16-merge" \ --delta ssf \ - --bf16 True \ --resume_peft "$DATA_PATH/llama2-7b-ssf-denas-bf16" \ --save_merged_model True \ - --denas "$DATA_PATH/llama2-7b-ssf-denas-bf16/best_model_structure.txt" \ + --merge_model_code_dir "instruction_tuning_pipeline/models/llama2-ssf" \ --debugs ``` -### 3. Evaluate merged model As ssf will enable bias, while the default Llama2 disable all the bias, to enable the full parameters of the adapter, we need to change the model definition. -First copy the updated model codes with the merged weights. -```shell -cp instruction_tuning_pipeline/models/llama2-ssf/* $DATA_PATH/llama2-7b-ssf-denas-bf16-merge/merged_model -``` +First, specify the `merge_model_code_dir` args, it will copy the updated model codes with the merged weights. -Then update the "best_model_structure" and "target_modules" setting in config.json. if not enable denas and not change "target_modules" default settings, can skip correpsonding setting. +Then it will automatically update the "best_model_structure" and "target_modules" setting in `config.json`. if "denas" or "target_modules" are not enabled/changed, it will skip the corresponding setting. +The changed `config.json` looks like this: ```shell ... "best_model_structure": {"num_hidden_layers": [1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1]}, # change to your best structure, skip to keep default @@ -78,6 +69,8 @@ Then update the "best_model_structure" and "target_modules" setting in config.js ... ``` +### 3. Evaluate merged model + Finally we can directly evalute the merged model. ```shell python instruction_tuning_pipeline/finetune_clm.py \ @@ -94,5 +87,4 @@ python instruction_tuning_pipeline/finetune_clm.py \ --trust_remote_code True \ --no_cuda \ --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16-merge/eval_merge" \ - --bf16 True ``` \ No newline at end of file diff --git a/tests/deltatuner/finetune/ssf/llama2-7b-ssf.sh b/tests/deltatuner/merge_model/ssf-merge-test.sh similarity index 51% rename from tests/deltatuner/finetune/ssf/llama2-7b-ssf.sh rename to tests/deltatuner/merge_model/ssf-merge-test.sh index 39299cf36..f1bfbcefe 100755 --- a/tests/deltatuner/finetune/ssf/llama2-7b-ssf.sh +++ b/tests/deltatuner/merge_model/ssf-merge-test.sh @@ -1,49 +1,43 @@ -DATA_PATH="/home/data" +DATA_PATH="/home/vmagent/app/data" -python instruction_tuning_pipeline/finetune_clm.py \ + +# merge model +python example/instruction_tuning_pipeline/finetune_clm.py \ --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ --train_file "$DATA_PATH/alpaca_data.json" \ --dataset_concatenation \ - --per_device_train_batch_size 8 \ --per_device_eval_batch_size 8 \ --gradient_accumulation_steps 1 \ --validation_split_percentage 30 \ - --do_train \ --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ --save_total_limit 1 \ --log_level info \ --save_strategy epoch \ --trust_remote_code True \ --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf" \ + --output_dir "$DATA_PATH/dtuner_test/models/llama2-7b-ssf-denas-bf16-merge" \ --delta ssf \ - --denas False \ - | tee llama2-7b-ssf-1epoch.log + --resume_peft "$DATA_PATH/dtuner_test/models/llama2-7b-ssf-denas-bf16" \ + --save_merged_model True \ + --merge_model_code_dir $DATA_PATH"/dtuner_test/ssf_models_code/llama-7b-ssf" \ + --bf16 True \ + --debugs -#bf16 -python instruction_tuning_pipeline/finetune_clm.py \ - --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \ + +# evaluate merged model +python example/instruction_tuning_pipeline/finetune_clm.py \ + --model_name_or_path $DATA_PATH"/dtuner_test/models/llama2-7b-ssf-denas-bf16-merge/merged_model" \ --train_file "$DATA_PATH/alpaca_data.json" \ --dataset_concatenation \ - --per_device_train_batch_size 8 \ --per_device_eval_batch_size 8 \ --gradient_accumulation_steps 1 \ --validation_split_percentage 30 \ - --do_train \ --do_eval \ - --learning_rate 1e-4 \ - --num_train_epochs 1 \ - --logging_steps 100 \ --save_total_limit 1 \ --log_level info \ - --save_strategy epoch \ --trust_remote_code True \ --no_cuda \ - --output_dir "$DATA_PATH/llama2-7b-ssf-bf16" \ - --delta ssf \ --bf16 True \ - --denas False \ - | tee llama2-7b-ssf-bf16-1epoch.log \ No newline at end of file + --output_dir $DATA_PATH"/dtuner_test/models/llama2-7b-ssf-denas-bf16-merge/eval_merge" \ + --debugs +