[v1.2][ISSUE-374] deltatuner denas-ssf multiple model validation and model merge

e2eAIOK/deltatuner/README.md

@@ -84,10 +84,10 @@ Please refer to [example page](https://github.com/intel/e2eAIOK/tree/main/exampl
 | Model        | LoRA | SSF  |
 |--------------| ---- | ---- |
 | GPT-2        | ✅  |  |
-| GPT-J        | ✅  |  |
-| Bloom        | ✅  |  |
-| OPT          | ✅  |  |
-| GPT-Neo      | ✅  |  |
-| Falcon       | ✅  |  |
+| GPT-J        | ✅  | ✅ |
+| Bloom        | ✅  | ✅ |
+| OPT          | ✅  | ✅ |
+| GPT-Neo      | ✅  | ✅ |
+| Falcon       | ✅  | ✅ |
 | LLaMA        | ✅  | ✅ |
 | MPT          | ✅  | ✅ |

e2eAIOK/deltatuner/deltatuner/deltatuner.py

@@ -39,9 +39,9 @@ def optimize(model, tokenizer, algo: str="auto", adapter_name: str="default", de
 
     if isinstance(peft_config, LoraConfig) or isinstance(peft_config, SSFConfig):
         if peft_config.task_type not in MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING:
-            model = DeltaTunerModel(model, tokenizer, peft_config, adapter_name, deltatuning_args)
+            model = DeltaTunerModel(model, peft_config, adapter_name, deltatuning_args, tokenizer)
         else:
-            model = MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING[peft_config.task_type](model, tokenizer, peft_config, adapter_name, deltatuning_args)
+            model = MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING[peft_config.task_type](model, peft_config, adapter_name, deltatuning_args, tokenizer)
         return model
     else:
         raise NotImplementedError("Current algorithm {} is not supported in deltatuner. ".format(algo))

e2eAIOK/deltatuner/deltatuner/deltatuner_model.py

@@ -52,7 +52,7 @@ def setup_seed(seed):
     torch.manual_seed(seed)
 
 class DeltaTunerModel(PeftModel, torch.nn.Module):
-    def __init__(self, model, tokenizer: AutoTokenizer, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None):
+    def __init__(self, model, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None, tokenizer: AutoTokenizer = None):
         torch.nn.Module.__init__(self)
         self.base_model = model
         self.tokenizer = tokenizer
@@ -104,7 +104,8 @@ def _init_denas_params_(self):
         self.denas_config.model_id = self.base_model.config._name_or_path
         self.denas_config.tokenizer = self.tokenizer
         self.denas_config.max_param_limits = sum(param.numel() for param in self.base_model.parameters() if param.requires_grad) / 10.**6 if self.denas_config.max_param_limits is None else self.denas_config.max_param_limits
-        self.denas_config.budget_latency_max = network_latency(self.base_model, self.tokenizer, batch_size=self.denas_config.batch_size) if self.denas_config.budget_latency_max is not None else self.denas_config.budget_latency_max
+        if self.tokenizer:
+            self.denas_config.budget_latency_max = network_latency(self.base_model, self.tokenizer, batch_size=self.denas_config.batch_size) if self.denas_config.budget_latency_max is not None else self.denas_config.budget_latency_max
 
     def search(self, denas_config, super_net, search_space):
         setup_seed(denas_config.random_seed)
@@ -219,6 +220,13 @@ def from_pretrained(
         from .mapping import DELTATUNER_TYPE_TO_CONFIG_MAPPING, MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING
         denas_config = kwargs.pop("denas_config", None)
 
+        best_structure_file = os.path.join(model_id, "best_model_structure.txt")
+        if os.path.isfile(best_structure_file):
+            denas_config.denas = True
+            denas_config.best_model_structure = best_structure_file
+        else:
+            denas_config.denas = False
+
         # load the config
         if config is None:
             peft_type_name = PeftConfig._get_peft_type(
@@ -376,8 +384,8 @@ def load_adapter(self, model_id: str, adapter_name: str, is_trainable: bool = Fa
         return load_result
 
 class DelatunerModelForCausalLM(DeltaTunerModel):
-    def __init__(self, model: PeftModel, tokenizer: AutoTokenizer, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None):
-        super().__init__(model, tokenizer, peft_config, adapter_name, denas_config)
+    def __init__(self, model: PeftModel, peft_config: PeftConfig, adapter_name: str = "default", denas_config: DeltaTunerArguments = None, tokenizer: AutoTokenizer = None):
+        super().__init__(model, peft_config, adapter_name, denas_config, tokenizer)
         self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation
 
     def forward(

e2eAIOK/deltatuner/deltatuner/tuner/deltatuner_ssf.py

@@ -1,3 +1,4 @@
+import os
 import math
 import re
 import warnings
@@ -80,6 +81,50 @@ class SSFConfig(PeftConfig):
     def __post_init__(self):
         self.peft_type = DeltaTunerType.SSF
 
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str, subfolder: Optional[str] = None, **kwargs):
+        r"""
+        This method loads the configuration of your adapter model from a directory.
+
+        Args:
+            pretrained_model_name_or_path (`str`):
+                The directory or the Hub repository id where the configuration is saved.
+            kwargs (additional keyword arguments, *optional*):
+                Additional keyword arguments passed along to the child class initialization.
+        """
+        from peft.utils import CONFIG_NAME
+
+        path = (
+            os.path.join(pretrained_model_name_or_path, subfolder)
+            if subfolder is not None
+            else pretrained_model_name_or_path
+        )
+
+        hf_hub_download_kwargs, class_kwargs, _ = cls._split_kwargs(kwargs)
+
+        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
+            config_file = os.path.join(path, CONFIG_NAME)
+        else:
+            try:
+                config_file = hf_hub_download(
+                    pretrained_model_name_or_path, CONFIG_NAME, subfolder=subfolder, **hf_hub_download_kwargs
+                )
+            except Exception:
+                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'")
+
+        loaded_attributes = cls.from_json_file(config_file)
+
+        config_cls = cls
+
+        config = config_cls(**class_kwargs)
+
+        for key, value in loaded_attributes.items():
+            if hasattr(config, key):
+                setattr(config, key, value)
+
+        return config
+
+
 class DeltaSSFSearchSpace:
     @classmethod
     def generate_search_space(cls, model, denas_config):

e2eAIOK/deltatuner/deltatuner/utils/config.py

@@ -1,11 +1,13 @@
 import enum
 from dataclasses import  field
 from typing import List, Optional, Tuple, Union
+from peft.utils import TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING
 
 class DeltaTunerType(str, enum.Enum):
     SSF = "SSF"
 
-TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING = {
+TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING
+TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING.update({
     "llama": ["q_proj", "v_proj"],
     "mpt": ["Wqkv","out_proj","up_proj","down_proj"]
-}
+})

example/instruction_tuning_pipeline/README.md

@@ -125,6 +125,10 @@ python example/instruction_tuning_pipeline/finetune_clm.py \
     --bf16 True 2>&1 | tee log/mpt-lora-run-1epoch.log
 ```
 
+- If you are using 4th Xeon or later (SPR etc.), please specify the `--bf16 --no_cuda` args;
+- If you are using 3th Xeon or before (ICX etc.): please specify the `--no_cuda` args;
+- If you are using GPU server: please specify the `--fp16` args.
+
 ## Evaluate the model
 
 For model evaluation, we follow the same method in [open_llm_leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard), which evaluate 4 key benchmarks in the [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness).

example/instruction_tuning_pipeline/finetune_clm.py

@@ -20,6 +20,8 @@
 import datasets
 import logging
 import os
+import json
+import errno
 import sys
 import transformers
 from transformers.modeling_utils import unwrap_model
@@ -292,6 +294,10 @@ class FinetuneArguments:
         default=False,
         metadata={"help": "save merged model"},
     )
+    merge_model_code_dir: Optional[str] = field(
+        default="",
+        metadata={"help": "the code path of base model with enable bias on target modules for ssf algo"},
+    )
 
 PROMPT_DICT = {
     "prompt_with_input": (
@@ -700,12 +706,7 @@ def concatenate_data(dataset, max_seq_length):
 
     if finetune_args.delta:
         if finetune_args.resume_peft != "":
-            if finetune_args.delta == "ssf":
-                peft_config = SSFConfig(target_modules=deltatuner_args.ssf_target_modules,
-                bias="none",
-                task_type="CAUSAL_LM",
-                )
-            model = DeltaTunerModel.from_pretrained(model, finetune_args.resume_peft, config=peft_config, denas_config=deltatuner_args)
+            model = DeltaTunerModel.from_pretrained(model, finetune_args.resume_peft, denas_config=deltatuner_args)
         else:
             model = deltatuner.optimize(model, tokenizer, algo=finetune_args.delta, deltatuning_args=deltatuner_args)
         logger.info("***deltatuner optimized model parameter***")
@@ -757,6 +758,37 @@ def concatenate_data(dataset, max_seq_length):
                     training_args.output_dir, state_dict=unwrapped_model.state_dict()
                 )
 
+    if finetune_args.save_merged_model:
+        if isinstance(model, PeftModel):
+            model = model.merge_and_unload()
+            saved_dir = os.path.join(training_args.output_dir, "merged_model")
+            os.makedirs(saved_dir, exist_ok=True)
+            print(f"copy base model config to {saved_dir}")
+            os.system(f"cp {model_args.model_name_or_path}/* {saved_dir}")
+            print(f"remove unnecessary file from {model_args.model_name_or_path}")
+            os.system(f"rm {saved_dir}/*.bin* {saved_dir}/*.safetensors*")
+            print(f"Save merged model to {saved_dir}")
+            torch.save(model.state_dict(), os.path.join(saved_dir, "pytorch_model.bin"))
+            if finetune_args.delta == 'ssf':
+                if os.path.exists(finetune_args.merge_model_code_dir):
+                    print(f"copy merged model code to {saved_dir}")
+                    os.system(f"cp {finetune_args.merge_model_code_dir}/* {saved_dir}")
+                else:
+                    raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), finetune_args.merge_model_code_dir)
+                pre_train_config_file = os.path.join(saved_dir, "config.json")
+                with open(pre_train_config_file, "r") as file:
+                    config_json = json.load(file)
+                adapter_config_file = os.path.join(finetune_args.resume_peft, "adapter_config.json")
+                with open(adapter_config_file, "r") as file:
+                    adapter_config_json = json.load(file)
+                config_json['target_modules'] = adapter_config_json['target_modules']
+
+                best_structure_file = os.path.join(finetune_args.resume_peft, "best_model_structure.txt")
+                if os.path.isfile(best_structure_file):
+                    with open(best_structure_file, "r") as file:
+                        best_structure_json = json.loads(file.readline().strip())
+                    config_json['best_model_structure'] = best_structure_json
+
     # Evaluation
     if training_args.do_eval:
         logger.info("*** Evaluate ***")
@@ -771,12 +803,6 @@ def concatenate_data(dataset, max_seq_length):
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
 
-    if finetune_args.save_merged_model:
-        if isinstance(model, PeftModel):
-            merged_model = model.merge_and_unload()
-            os.makedirs(os.path.join(training_args.output_dir,"merged_model"),exist_ok=True)
-            torch.save(merged_model.state_dict(),os.path.join(training_args.output_dir,"merged_model","pytorch_model.bin"))
-            print(f"Save merged model to {training_args.output_dir}")
 
 def profile_model(model, train_dataset, data_collator, args):
     from transformers import get_scheduler

tests/deltatuner/cicd/run_denas_lora.sh

@@ -7,56 +7,41 @@ pip uninstall wandb -y
 
 cd /home/vmagent/app/e2eaiok
 
-mkdir -p log models
-
 DATA_PATH="/home/vmagent/app/data"
+LOG_PATH=$DATA_PATH"/dtuner_test/log"
+MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models"
 
-# fine-tune mpt-7b with denas-lora
-python example/instruction_tuning_pipeline/finetune_clm.py \
-    --model_name_or_path $DATA_PATH"/mpt-7b" \
-    --train_file $DATA_PATH"/alpaca_data.json" \
-    --dataset_concatenation \
-    --per_device_train_batch_size 8 \
-    --per_device_eval_batch_size 8 \
-    --gradient_accumulation_steps 1 \
-    --do_train \
-    --do_eval \
-    --validation_split_percentage 30 \
-    --learning_rate 1e-4 \
-    --num_train_epochs 1 \
-    --logging_steps 100 \
-    --save_total_limit 1 \
-    --log_level info \
-    --save_strategy epoch \
-    --output_dir models/mpt_denas-lora_model \
-    --peft lora \
-    --delta lora \
-    --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \
-    --trust_remote_code True \
-    --no_cuda \
-    2>&1 | tee log/mpt-denas-lora-run-1epoch.log
+mkdir -p $LOG_PATH $MODEL_SAVE_PATH
 
-# fine-tune llama2-7b with denas-lora
-python example/instruction_tuning_pipeline/finetune_clm.py \
-    --model_name_or_path $DATA_PATH"/Llama-2-7b-hf" \
-    --train_file $DATA_PATH"/alpaca_data.json" \
-    --dataset_concatenation \
-    --per_device_train_batch_size 8 \
-    --per_device_eval_batch_size 8 \
-    --gradient_accumulation_steps 1 \
-    --do_train \
-    --do_eval \
-    --validation_split_percentage 30 \
-    --learning_rate 1e-4 \
-    --num_train_epochs 1 \
-    --logging_steps 100 \
-    --save_total_limit 1 \
-    --log_level info \
-    --save_strategy epoch \
-    --output_dir models/llama2_denas-lora_model \
-    --peft lora \
-    --delta lora \
-    --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \
-    --trust_remote_code True \
-    --no_cuda \
-    2>&1 | tee log/llama2-denas-lora-run-1epoch.log
+# fine-tune with denas-lora
+model_name_list="mpt-7b Llama-2-7b-hf"
+for model_name in $model_name_list
+do
+    model_name_or_path=${DATA_PATH}"/"${model_name}
+    model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_denas-lora"
+    log_save_path=$LOG_PATH"/"${model_name}"_denas-lora-1epoch.log"
+    python example/instruction_tuning_pipeline/finetune_clm.py \
+        --model_name_or_path $model_name_or_path \
+        --train_file $DATA_PATH"/alpaca_data.json" \
+        --dataset_concatenation \
+        --per_device_train_batch_size 8 \
+        --per_device_eval_batch_size 8 \
+        --gradient_accumulation_steps 1 \
+        --do_train \
+        --do_eval \
+        --validation_split_percentage 30 \
+        --learning_rate 1e-4 \
+        --num_train_epochs 1 \
+        --logging_steps 100 \
+        --save_total_limit 1 \
+        --log_level info \
+        --save_strategy epoch \
+        --output_dir $model_save_path \
+        --peft lora \
+        --delta lora \
+        --denas True \
+        --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \
+        --trust_remote_code True \
+        --no_cuda \
+        2>&1 | tee $log_save_path
+done

tests/deltatuner/cicd/run_denas_ssf.sh

@@ -6,38 +6,21 @@ pip uninstall wandb -y
 
 cd /home/vmagent/app/e2eaiok
 
-mkdir -p log
-
 DATA_PATH="/home/vmagent/app/data"
+LOG_PATH=$DATA_PATH"/dtuner_test/log"
+MODEL_SAVE_PATH=$DATA_PATH"/dtuner_test/models"
 
-#run mpt with ssf and denas, bf16
-python example/instruction_tuning_pipeline/finetune_clm.py \
-        --model_name_or_path "$DATA_PATH/mpt-7b" \
-        --train_file "$DATA_PATH/alpaca_data.json" \
-        --dataset_concatenation \
-        --per_device_train_batch_size 8 \
-        --per_device_eval_batch_size 8 \
-        --gradient_accumulation_steps 1 \
-        --validation_split_percentage 30 \
-        --do_train \
-        --do_eval \
-        --learning_rate 1e-4 \
-        --num_train_epochs 1 \
-        --logging_steps 100 \
-        --save_total_limit 1 \
-        --log_level info \
-        --save_strategy epoch \
-        --trust_remote_code True \
-        --no_cuda \
-        --output_dir "$DATA_PATH/mpt-7b-ssf-allmodules-denas-bf16" \
-        --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \
-        --delta ssf \
-        --denas True \
-        2>&1 | tee "log/mpt-7b-ssf-allmodules-denas-bf16-run-1epoch.log"
+mkdir -p $LOG_PATH $MODEL_SAVE_PATH
 
-#run llama with ssf and denas, bf16
-python example/instruction_tuning_pipeline/finetune_clm.py \
-        --model_name_or_path "$DATA_PATH/Llama-2-7b-hf" \
+# fine-tune with denas-ssf
+model_name_list="mpt-7b Llama-2-7b-hf"
+for model_name in $model_name_list
+do
+    model_name_or_path=${DATA_PATH}"/"${model_name}
+    model_save_path=${MODEL_SAVE_PATH}"/"${model_name}"_denas-ssf"
+    log_save_path=$LOG_PATH"/"${model_name}"_denas-ssf-1epoch.log"
+    python example/instruction_tuning_pipeline/finetune_clm.py \
+        --model_name_or_path $model_name_or_path \
         --train_file "$DATA_PATH/alpaca_data.json" \
         --dataset_concatenation \
         --per_device_train_batch_size 8 \
@@ -54,8 +37,9 @@ python example/instruction_tuning_pipeline/finetune_clm.py \
         --save_strategy epoch \
         --trust_remote_code True \
         --no_cuda \
-        --output_dir "$DATA_PATH/llama2-7b-ssf-denas-bf16" \
+        --output_dir $model_save_path \
         --debugs --max_epochs 1 --population_num 1 --crossover_num 1 --mutation_num 1 --select_num 1 \
         --delta ssf \
         --denas True \
-        2>&1 | tee log/llama2-7b-ssf-denas-bf16-1epoch.log
+        2>&1 | tee $log_save_path
+done