Skip to content

Commit

Permalink
optim e2e tests to run a bit faster (#2069) [skip ci]
Browse files Browse the repository at this point in the history 
* optim e2e tests to run a bit faster

* run prequant w/o lora_modules_to_save

* use smollm2
  • Loading branch information
@winglian
winglian authored Nov 18, 2024
1 parent fb346fa commit 6d7e794
Show file tree
Hide file tree
Showing 6 changed files with 43 additions and 23 deletions.
19 changes: 19 additions & 0 deletions tests/e2e/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,25 @@
import tempfile

import pytest
from huggingface_hub import snapshot_download


@pytest.fixture(scope="session", autouse=True)
def download_smollm2_135m_model():
# download the model
snapshot_download("HuggingFaceTB/SmolLM2-135M")


@pytest.fixture(scope="session", autouse=True)
def download_tatsu_lab_alpaca_dataset():
# download the model
snapshot_download("tatsu-lab/alpaca", repo_type="dataset")


@pytest.fixture(scope="session", autouse=True)
def download_mhenrichsen_alpaca_2k_dataset():
# download the model
snapshot_download("mhenrichsen/alpaca_2k_test", repo_type="dataset")


@pytest.fixture
Expand Down
38 changes: 17 additions & 21 deletions tests/e2e/multigpu/test_llama.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
@pytest.fixture(scope="session", autouse=True)
def download_model():
# download the model
snapshot_download("TinyLlama/TinyLlama_v1.1")
snapshot_download("HuggingFaceTB/SmolLM2-135M")


class TestMultiGPULlama:
Expand All @@ -37,7 +37,7 @@ def test_lora_ddp(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 2048,
"adapter": "lora",
"lora_r": 8,
Expand Down Expand Up @@ -93,7 +93,7 @@ def test_lora_ddp_packed(self, temp_dir, gradient_accumulation_steps):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 2048,
"sample_packing": True,
"eval_sample_packing": False,
Expand Down Expand Up @@ -149,8 +149,7 @@ def test_dpo_lora_ddp(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "TinyLlama/TinyLlama_v1.1",
"tokenizer_type": "LlamaTokenizer",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 2048,
"sample_packing": False,
"eval_sample_packing": False,
Expand All @@ -163,12 +162,10 @@ def test_dpo_lora_ddp(self, temp_dir):
"lora_target_linear": True,
"val_set_size": 0.05,
"special_tokens": {
"unk_token": "<unk>",
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<|endoftext|>",
},
"rl": "dpo",
"chat_template": "llama3",
"chat_template": "chatml",
"datasets": [
{
"path": "fozziethebeat/alpaca_messages_2k_dpo_test",
Expand Down Expand Up @@ -221,7 +218,7 @@ def test_dpo_qlora_ddp(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 2048,
"sample_packing": False,
"eval_sample_packing": False,
Expand Down Expand Up @@ -294,7 +291,7 @@ def test_fsdp(self, temp_dir, gradient_accumulation_steps):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 2048,
"val_set_size": 0.01,
"special_tokens": {
Expand Down Expand Up @@ -359,7 +356,7 @@ def test_fsdp_packed(self, temp_dir, fsdp_state_dict_type):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sample_packing": True,
"pad_to_sequence_len": True,
"sequence_len": 2048,
Expand Down Expand Up @@ -422,26 +419,25 @@ def test_fsdp_qlora_prequant_packed(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "axolotl-ai-co/TinyLlama_v1.1-bnb-nf4-bf16",
"tokenizer_type": "AutoTokenizer",
"base_model": "axolotl-ai-co/SmolLM2-135M-bnb-nf4-bf16",
"adapter": "qlora",
"mean_resizing_embeddings": True,
"load_in_4bit": True,
"lora_r": 8,
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_target_linear": True,
"lora_modules_to_save": [
"embed_tokens",
"lm_head",
],
# "lora_modules_to_save": [
# "embed_tokens",
# "lm_head",
# ],
"sample_packing": True,
"eval_sample_packing": False,
"pad_to_sequence_len": True,
"sequence_len": 2048,
"val_set_size": 0.05,
"special_tokens": {
"pad_token": "</s>",
"pad_token": "<|endoftext|>",
},
"datasets": [
{
Expand Down Expand Up @@ -503,7 +499,7 @@ def test_ds_zero3_packed(self, temp_dir, gradient_accumulation_steps):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sample_packing": True,
"pad_to_sequence_len": True,
"sequence_len": 2048,
Expand Down Expand Up @@ -553,7 +549,7 @@ def test_ds_zero3_qlora_packed(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"load_in_4bit": True,
"adapter": "qlora",
"lora_r": 8,
Expand Down
2 changes: 2 additions & 0 deletions tests/e2e/patched/test_fa_xentropy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ def test_lora_packing_fa_cross_entropy(self, temp_dir):
},
],
"num_epochs": 1,
"max_steps": 10,
"save_steps": 10,
"micro_batch_size": 8,
"gradient_accumulation_steps": 1,
"output_dir": temp_dir,
Expand Down
3 changes: 3 additions & 0 deletions tests/e2e/patched/test_lora_llama_multipack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ def test_lora_packing(self, temp_dir):
},
],
"num_epochs": 2,
"max_steps": 20,
"save_steps": 10,
"micro_batch_size": 8,
"gradient_accumulation_steps": 1,
"output_dir": temp_dir,
Expand Down Expand Up @@ -109,6 +111,7 @@ def test_lora_gptq_packed(self, temp_dir):
},
],
"num_epochs": 2,
"max_steps": 20,
"save_steps": 0.5,
"micro_batch_size": 8,
"gradient_accumulation_steps": 1,
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/test_optimizers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def test_adopt_adamw(self, temp_dir):
def test_fft_schedule_free_adamw(self, temp_dir):
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 1024,
"val_set_size": 0.1,
"special_tokens": {
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/test_packing_loss.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def test_loss_packed(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "HuggingFaceTB/SmolLM-135M",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 1024,
"sample_packing": True,
"flash_attention": True,
Expand Down

0 comments on commit 6d7e794

Please sign in to comment.