[v1.2][ISSUE-413]Dtuner prepare for integration

.github/workflows/e2eaiok_deltatuner_release_pypi.yml

@@ -1,4 +1,4 @@
-name: Publish E2EAIOK Deltatuner nightly to PyPI
+name: Publish E2EAIOK Deltatuner Release to PyPI
 
 on:
   workflow_dispatch:

.github/workflows/integration_test_deltatuner.yml

@@ -13,6 +13,9 @@ on:
 jobs:
   integration_test:
     name: Integration Test Deltatuner
+    strategy:
+      matrix:
+        algo: [ lora, denas-lora, ssf, denas-ssf ]
     runs-on: self-hosted
     steps:
     - name: Checkout
@@ -26,24 +29,24 @@ jobs:
 
     - name : Test Lora
       run: |
-        if [[ $(docker ps -q --filter "name=test-deltatuner" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-deltatuner" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
-        docker run --rm --name test-deltatuner --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
+        if [[ $(docker ps -q --filter "name=${{ matrix.algo }}" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=${{ matrix.algo }}" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
+        docker run --rm --name ${{ matrix.algo }} --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
         /bin/bash -c ". /home/vmagent/app/e2eaiok/tests/deltatuner/cicd/run_lora.sh"
 
-    - name : Test SSF
-      run: |
-        if [[ $(docker ps -q --filter "name=test-deltatuner" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-deltatuner" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
-        docker run --rm --name test-deltatuner --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
-        /bin/bash -c ". /home/vmagent/app/e2eaiok/tests/deltatuner/cicd/run_ssf.sh"
-
     - name : Test DENAS Lora
       run: |
-        if [[ $(docker ps -q --filter "name=test-deltatuner" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-deltatuner" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
-        docker run --rm --name test-deltatuner --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
+        if [[ $(docker ps -q --filter "name=${{ matrix.algo }}" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=${{ matrix.algo }}" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
+        docker run --rm --name ${{ matrix.algo }} --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
         /bin/bash -c ". /home/vmagent/app/e2eaiok/tests/deltatuner/cicd/run_denas_lora.sh"
 
+    - name : Test SSF
+      run: |
+        if [[ $(docker ps -q --filter "name=${{ matrix.algo }}" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=${{ matrix.algo }}" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
+        docker run --rm --name ${{ matrix.algo }} --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
+        /bin/bash -c ". /home/vmagent/app/e2eaiok/tests/deltatuner/cicd/run_ssf.sh"
+
     - name : Test DENAS SSF
       run: |
-        if [[ $(docker ps -q --filter "name=test-deltatuner" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-deltatuner" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
-        docker run --rm --name test-deltatuner --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
-        /bin/bash -c ". /home/vmagent/app/e2eaiok/tests/deltatuner/cicd/run_denas_ssf.sh"
+        if [[ $(docker ps -q --filter "name=${{ matrix.algo }}" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=${{ matrix.algo }}" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
+        docker run --rm --name ${{ matrix.algo }} --shm-size=100g --privileged --network host --device=/dev/dri -v /mnt/DP_disk2/dataset:/home/vmagent/app/data  -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ chatbot_finetune \
+        /bin/bash -c ". /home/vmagent/app/e2eaiok/tests/deltatuner/cicd/run_denas_ssf.sh"

.gitignore

@@ -6,3 +6,4 @@
 **/core.*
 **/__pycache__
 **/.ipynb_checkpoints
+**/dist

Dockerfile-ubuntu/Dockerfile-v1.2

@@ -12,9 +12,9 @@ RUN apt-get -y update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip install torch==1.13.0 transformers datasets accelerate SentencePiece \
+RUN pip install torch>=1.13.1 transformers datasets SentencePiece \
     evaluate peft==0.4.0 torchsummary \
-    nltk rouge_score protobuf==3.20.1 tokenizers einops wandb \
+    nltk rouge_score einops wandb \
     sigopt
 
 RUN pip install --upgrade pip

e2eAIOK/deltatuner/README.md

@@ -57,16 +57,19 @@ deltatuner_model = deltatuner.optimize(model=lora_model, tokenizer=tokenizer, de
 ### API reference
 In above examples, `deltatuner.optimize` is a python function to using deltatuner supported optimization algorithms to optimize the model.
 ```python
-def optimize(model, tokenizer, algo: str="auto", deltatuning_args: DeltaTunerArguments=None) -> DeltaTunerModel:
+def optimize(model, tokenizer, algo: str="auto", deltatuning_args: DeltaTunerArguments=None, **kwargs) -> DeltaTunerModel:
     '''
     Parameters:
-        model  - a PreTrainedModel or LoraModel. Specifies what model should be optimized
-        tokenizer - a tokenizer for preprocess text
-        algo (str, optional) – the algorithm. Specifies what type of adapter algorithm (default: “auto”)
-            "auto" – If the input model is mpt, the algorithm is ssf; elif the algorithm is lora
-            "lora" – use the lora algotihm
-            "ssf" – use the ssf algotithm
-        deltatuning_args (optional) – the deltatuner configuration. Specifically, deltatuning_args.denas is to use the denas in the optimization (default: True)
+        - model - a PreTrainedModel or LoraModel. Specifies what model should be optimized
+        - tokenizer - a tokenizer for preprocess text
+        - deltatuning_args (optional) – the deltatuner configuration. 
+          - deltatuning_args.denas is to use the denas in the optimization (default: True)
+          - deltatuning_args.algo Specifies what type of adapter algorithm (default: auto)
+            - "auto" – If the input model is mpt, the algorithm is ssf; elif the algorithm is lora
+            - "lora" – use the lora algotihm
+            - "ssf" – use the ssf algotithm 
+          - deltatuning_args.best_model_structure Specifies the pre-searched delta best structure so the model can be directly initilized without searching.
+        - kwargs - used to initilize deltatuning_args through key=value, such as algo="lora"
     Return 
         DeltaTunerModel - a wrapper of model, which composed of the original properties/function together with adavance properties/function provided by deltatuner
     '''
@@ -78,6 +81,7 @@ def optimize(model, tokenizer, algo: str="auto", deltatuning_args: DeltaTunerArg
 Please refer to [example page](https://github.com/intel/e2eAIOK/tree/main/example) for more use cases on fine-tuning other LLMs with the help of DeltaTuner.
 
 ## Model supported matrix
+We have upload the searched delta best structure to the [conf dir](https://github.com/intel/e2eAIOK/tree/main/e2eAIOK/deltatuner/deltatuner/conf/best_structure), so that users can directly use our searched structure for directly fine-tuning by passing the `DeltaTunerArguments.best_model_structure` to the `deltatuner.optimize` function.
 
 ### Causal Language Modeling
 

e2eAIOK/deltatuner/deltatuner/conf/best_structure/EleutherAI/gpt-j-6b-best_structure.jsonl

@@ -0,0 +1 @@
+{"num_hidden_layers": [1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1], "r": [11, 9, 11, 9, 12, 1, 8, 6, 11, 3, 5, 10, 8, 3, 5, 7, 9, 6, 7, 9, 12, 6, 5, 12, 1, 12, 12, 9], "alpha": [6, 6, 6, 3, 6, 1, 6, 6, 5, 6, 2, 4, 6, 6, 6, 6, 5, 6, 3, 2, 6, 5, 5, 6, 2, 1, 3, 3]}

e2eAIOK/deltatuner/deltatuner/conf/best_structure/bigscience/bloom-560m-best_structure.jsonl

@@ -0,0 +1 @@
+{"num_hidden_layers": [1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1], "r": [5, 8, 5, 3, 12, 8, 9, 4, 12, 1, 9, 8, 12, 12, 3, 12, 12, 9, 12, 3, 12, 9, 4, 7], "alpha": [4, 5, 6, 2, 6, 2, 5, 6, 4, 3, 1, 5, 6, 5, 4, 2, 6, 1, 6, 3, 6, 2, 4, 6]}

e2eAIOK/deltatuner/deltatuner/conf/best_structure/facebook/opt-125m-best_structure.jsonl

@@ -0,0 +1 @@
+{"num_hidden_layers": [1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1], "r": [8, 7, 7, 7, 6, 2, 5, 12, 10, 9, 2, 10], "alpha": [5, 6, 1, 6, 5, 4, 4, 6, 5, 6, 1, 4]}

e2eAIOK/deltatuner/deltatuner/conf/best_structure/gpt2-best_structure.jsonl

@@ -0,0 +1 @@
+{"num_hidden_layers": [1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1], "r": [12, 11, 10, 11, 3, 7, 12, 7, 7, 8, 7, 11], "alpha": [5, 6, 4, 5, 5, 6, 2, 2, 4, 2, 6, 6]}

e2eAIOK/deltatuner/deltatuner/conf/best_structure/meta-llama/Llama-2-7b-hf-best_structure.jsonl

@@ -0,0 +1 @@
+{"num_hidden_layers": [1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1], "r": [12, 12, 7, 9, 12, 10, 8, 7, 5, 5, 3, 6, 3, 2, 5, 6, 11, 2, 1, 4, 5, 3, 7, 3, 1, 1, 8, 12, 10, 11, 5, 12], "alpha": [5, 5, 6, 2, 2, 6, 6, 5, 1, 6, 5, 6, 4, 5, 4, 5, 1, 1, 6, 4, 6, 3, 5, 1, 5, 5, 4, 2, 5, 6, 5, 4]}

e2eAIOK/deltatuner/deltatuner/conf/best_structure/mosaicml/mpt-7b-best_structure.jsonl

@@ -0,0 +1 @@
+{"num_hidden_layers": [1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1], "r": [11, 12, 6, 11, 12, 5, 5, 7, 11, 12, 9, 9, 9, 2, 12, 8, 11, 3, 11, 7, 9, 5, 6, 11, 8, 2, 10, 12, 10, 7, 9, 4], "alpha": [1, 5, 1, 3, 4, 5, 5, 2, 6, 5, 3, 5, 5, 1, 6, 6, 4, 3, 5, 5, 2, 6, 2, 5, 2, 1, 6, 4, 6, 6, 2, 6]}

e2eAIOK/deltatuner/deltatuner/deltatuner_model.py

@@ -29,7 +29,7 @@
 from .tuner import DeltaLoraModel, DeltaLoraSearchSpace, DeltaSSFModel, DeltaSSFSearchSpace
 from .search import SearchEngineFactory, Timer
 from .search.utils import network_latency
-from .utils import DeltaTunerType, get_deltatuner_model_state_dict, set_deltatuner_model_state_dict
+from .utils import DeltaTunerType, get_deltatuner_model_state_dict, set_deltatuner_model_state_dict, BEST_MODEL_STRUCTURE_DEFAULT_NAME
 from typing import Any, Dict, List, Optional, Union
 
 DELTATUNNER_TO_MODEL_MAPPING = {
@@ -66,7 +66,7 @@ def __init__(self, model, peft_config: PeftConfig, adapter_name: str = "default"
         logging.basicConfig(level=logging.INFO,
                 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
         self.logger = logging.getLogger('deltatuner')
-        if not isinstance(model, PeftModel):
+        if peft_config.peft_type in (DeltaTunerType.SSF):
             self.base_model = DELTATUNNER_TO_MODEL_MAPPING[peft_config.peft_type](
                 self.base_model, self.peft_config, adapter_name
             )
@@ -203,7 +203,7 @@ def save_pretrained(
             peft_config.inference_mode = inference_mode
 
         if self.best_model_param is not None:
-            with open(os.path.join(save_directory, "best_model_structure.txt"), "w+") as fout:
+            with open(os.path.join(save_directory, BEST_MODEL_STRUCTURE_DEFAULT_NAME), "w+") as fout:
                 fout.write(json.dumps(self.best_model_param))
 
     @classmethod
@@ -218,14 +218,7 @@ def from_pretrained(
     ):
         from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING
         from .mapping import DELTATUNER_TYPE_TO_CONFIG_MAPPING, MODEL_TYPE_TO_DELTATUNER_MODEL_MAPPING
-        denas_config = kwargs.pop("denas_config", None)
-
-        best_structure_file = os.path.join(model_id, "best_model_structure.txt")
-        if os.path.isfile(best_structure_file):
-            denas_config.denas = True
-            denas_config.best_model_structure = best_structure_file
-        else:
-            denas_config.denas = False
+        denas_config = cls._get_denas_config(model_id, **kwargs)
 
         # load the config
         if config is None:
@@ -262,6 +255,46 @@ def from_pretrained(
         model.load_adapter(model_id, adapter_name, is_trainable=is_trainable, **kwargs)
         return model
 
+    @classmethod
+    def _get_denas_config(cls, model_id: str, **kwargs: Any):
+        hf_hub_download_kwargs, kwargs = cls._split_kwargs(kwargs)
+
+        # load weights if any
+        path = (
+            os.path.join(model_id, hf_hub_download_kwargs["subfolder"])
+            if hf_hub_download_kwargs.get("subfolder", None) is not None
+            else model_id
+        )
+
+        if os.path.exists(os.path.join(path, BEST_MODEL_STRUCTURE_DEFAULT_NAME)):
+            filename = os.path.join(path, BEST_MODEL_STRUCTURE_DEFAULT_NAME)
+        else:
+            has_remote_structure_file = hub_file_exists(
+                model_id,
+                BEST_MODEL_STRUCTURE_DEFAULT_NAME,
+                revision=hf_hub_download_kwargs.get("revision", None),
+                repo_type=hf_hub_download_kwargs.get("repo_type", None),
+            )
+
+            if has_remote_structure_file:
+                filename = hf_hub_download(
+                    model_id,
+                    BEST_MODEL_STRUCTURE_DEFAULT_NAME,
+                    **hf_hub_download_kwargs,
+                )
+            else:
+                raise ValueError(
+                    f"Can't find structure for {model_id} in {model_id} or in the Hugging Face Hub. "
+                    f"Please check that the file {BEST_MODEL_STRUCTURE_DEFAULT_NAME} is present at {model_id}."
+                )
+
+        denas_config = DeltaTunerArguments()
+        denas_config.denas = True
+        denas_config.best_model_structure = filename
+
+        return denas_config
+
+
     def load_adapter(self, model_id: str, adapter_name: str, is_trainable: bool = False, **kwargs: Any):
         from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING
         from .mapping import DELTATUNER_TYPE_TO_CONFIG_MAPPING

e2eAIOK/deltatuner/deltatuner/utils/__init__.py

@@ -1,2 +1,2 @@
-from .config import DeltaTunerType, TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING
+from .config import DeltaTunerType, TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING, BEST_MODEL_STRUCTURE_DEFAULT_NAME
 from .save_and_load import get_deltatuner_model_state_dict, set_deltatuner_model_state_dict

e2eAIOK/deltatuner/deltatuner/utils/config.py

@@ -10,4 +10,6 @@ class DeltaTunerType(str, enum.Enum):
 TRANSFORMERS_MODELS_TO_SSF_TARGET_MODULES_MAPPING.update({
     "llama": ["q_proj", "v_proj"],
     "mpt": ["Wqkv","out_proj","up_proj","down_proj"]
-})
+})
+
+BEST_MODEL_STRUCTURE_DEFAULT_NAME = "best_model_structure.txt"

e2eAIOK/deltatuner/deltatuner/version

@@ -1 +1 @@
-1.1.0
+1.1.8

e2eAIOK/deltatuner/docker/Dockerfile

@@ -52,7 +52,7 @@ RUN conda init bash && \
     conda create -yn chatbot-finetuning python=3.9 && \
     echo "conda activate chatbot-finetuning" >> ~/.bashrc && \
     source ~/.bashrc && \
-    pip install torch==1.13.0 transformers datasets accelerate SentencePiece \
+    pip install torch>=1.13.1 transformers datasets SentencePiece \
     evaluate peft==0.4.0 torchsummary \
-    nltk rouge_score protobuf==3.20.1 tokenizers einops wandb \
+    nltk rouge_score einops wandb \
     sigopt

e2eAIOK/deltatuner/docker/DockerfileGPU

@@ -40,7 +40,7 @@ RUN conda init bash && \
     source ~/.bashrc && \
     pip install transformers datasets accelerate SentencePiece \
     peft==0.4.0 torchsummary sigopt evaluate \
-    nltk rouge_score protobuf==3.20.1 tokenizers einops \
+    nltk rouge_score tokenizers einops \
     wandb
 
 RUN git clone https://github.com/EleutherAI/lm-evaluation-harness.git ~/lm-evaluation-harness && \

e2eAIOK/deltatuner/setup.py

@@ -7,7 +7,7 @@
     VERSION = fh.read().strip()
 
 REQUIRED_PACKAGES = [
-    'torch>=1.13.0', 'transformers', 'datasets', 'sentencepiece', 
+    'torch>=1.13.1', 'transformers', 'datasets', 'sentencepiece', 
     'peft==0.4.0', 'evaluate', 'nltk', 'rouge_score', 'einops', 
     'sigopt', 'torchsummary'
 ]
@@ -24,7 +24,7 @@
     download_url='https://github.com/intel/e2eAIOK/',
     packages=setuptools.find_packages(
         exclude=["example", "docker", ]),
-    package_data={'deltatuner': ['version']}, 
+    package_data={'deltatuner': ['version', '*/*/*', '*/*/*/*'], }, 
     python_requires=">=3.7",  # '>=3.4',  # !=3.4.*
     install_requires=REQUIRED_PACKAGES,
     extras_require={

example/instruction_tuning_pipeline/finetune_clm.py

@@ -699,7 +699,7 @@ def concatenate_data(dataset, max_seq_length):
 
     if deltatuner_args.algo:
         if finetune_args.resume_peft != "":
-            model = DeltaTunerModel.from_pretrained(model, finetune_args.resume_peft, denas_config=deltatuner_args)
+            model = DeltaTunerModel.from_pretrained(model, finetune_args.resume_peft)
         else:
             model = deltatuner.optimize(model, tokenizer, deltatuning_args=deltatuner_args)
         logger.info("***deltatuner optimized model parameter***")