Skip to content

Commit

Permalink
Update Bert-Large PyTorch batch sizes (#12892)
Browse files Browse the repository at this point in the history 
Update Bert-Large PyTorch batch sizes to use MLPerf configs:
https://github.com/mlcommons/inference_results_v2.1/tree/master/closed/NVIDIA/configs/bert

No effect on benchmark running time. CUDA <10 mins and x86 < 20 mins.
  • Loading branch information
@mariecwhite
mariecwhite authored Apr 3, 2023
1 parent be0f1e1 commit 92cc535
Show file tree
Hide file tree
Showing 5 changed files with 117 additions and 70 deletions.
6 changes: 3 additions & 3 deletions build_tools/python/e2e_test_framework/models/model_groups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@
# BERT-Large in various batch sizes.
BERT_LARGE_TORCH_BATCHES = [
torch_models.BERT_LARGE_1X384_FP32_TORCH,
torch_models.BERT_LARGE_8X384_FP32_TORCH,
torch_models.BERT_LARGE_16X384_FP32_TORCH,
torch_models.BERT_LARGE_24X384_FP32_TORCH,
torch_models.BERT_LARGE_32X384_FP32_TORCH,
torch_models.BERT_LARGE_48X384_FP32_TORCH,
torch_models.BERT_LARGE_64X384_FP32_TORCH,
# Disabled due to https://github.com/openxla/iree/issues/12774.
#torch_models.BERT_LARGE_128X384_FP32_TORCH,
#torch_models.BERT_LARGE_256X384_FP32_TORCH,
#torch_models.BERT_LARGE_512X384_FP32_TORCH,
#torch_models.BERT_LARGE_1024X384_FP32_TORCH,
#torch_models.BERT_LARGE_1280X384_FP32_TORCH,
]

RESNET50_TORCH_BATCHES = [
Expand Down
72 changes: 36 additions & 36 deletions build_tools/python/e2e_test_framework/models/torch_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,77 +82,67 @@
tags=["fp32", "transformer", "seqlen384", "batch-1"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_1/linalg.mlir",
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_1/linalg.mlir",
entry_function="forward",
input_types=["1x384xi64", "1x384xi64"])

BERT_LARGE_8X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_8X384_FP32_TORCH,
name="BertLargePTBatch8",
tags=["fp32", "transformer", "seqlen384", "batch-8"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_8/linalg.mlir",
entry_function="forward",
input_types=["8x384xi64", "8x384xi64"])

BERT_LARGE_16X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_16X384_FP32_TORCH,
name="BertLargePTBatch16",
tags=["fp32", "transformer", "seqlen384", "batch-16"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_16/linalg.mlir",
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_16/linalg.mlir",
entry_function="forward",
input_types=["16x384xi64", "16x384xi64"])

BERT_LARGE_24X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_24X384_FP32_TORCH,
name="BertLargePTBatch24",
tags=["fp32", "transformer", "seqlen384", "batch-24"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_24/linalg.mlir",
entry_function="forward",
input_types=["24x384xi64", "24x384xi64"])

BERT_LARGE_32X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_32X384_FP32_TORCH,
name="BertLargePTBatch32",
tags=["fp32", "transformer", "seqlen384", "batch-32"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_32/linalg.mlir",
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_32/linalg.mlir",
entry_function="forward",
input_types=["32x384xi64", "32x384xi64"])

BERT_LARGE_48X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_48X384_FP32_TORCH,
name="BertLargePTBatch48",
tags=["fp32", "transformer", "seqlen384", "batch-48"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_48/linalg.mlir",
entry_function="forward",
input_types=["48x384xi64", "48x384xi64"])

BERT_LARGE_64X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_64X384_FP32_TORCH,
name="BertLargePTBatch64",
tags=["fp32", "transformer", "seqlen384", "batch-64"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_64/linalg.mlir",
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_64/linalg.mlir",
entry_function="forward",
input_types=["64x384xi64", "64x384xi64"])

BERT_LARGE_128X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_128X384_FP32_TORCH,
name="BertLargePTBatch128",
tags=["fp32", "transformer", "seqlen384", "batch-128"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_128/linalg.mlir",
entry_function="forward",
input_types=["128x384xi64", "128x384xi64"])

BERT_LARGE_256X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_256X384_FP32_TORCH,
name="BertLargePTBatch256",
tags=["fp32", "transformer", "seqlen384", "batch-256"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_256/linalg.mlir",
entry_function="forward",
input_types=["256x384xi64", "256x384xi64"])

BERT_LARGE_512X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_512X384_FP32_TORCH,
name="BertLargePTBatch512",
tags=["fp32", "transformer", "seqlen384", "batch-512"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_512/linalg.mlir",
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_512/linalg.mlir",
entry_function="forward",
input_types=["512x384xi64", "512x384xi64"])

Expand All @@ -162,10 +152,20 @@
tags=["fp32", "transformer", "seqlen384", "batch-1024"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_1024/linalg.mlir",
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_1024/linalg.mlir",
entry_function="forward",
input_types=["1024x384xi64", "1024x384xi64"])

BERT_LARGE_1280X384_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_BERT_LARGE_1280X384_FP32_TORCH,
name="BertLargePTBatch1280",
tags=["fp32", "transformer", "seqlen384", "batch-1280"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_1280/linalg.mlir",
entry_function="forward",
input_types=["1280x384xi64", "1280x384xi64"])

# Converted from https://pytorch.org/vision/main/models/generated/torchvision.models.resnet50.html
RESNET50_1X3X224X224_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_RESNET50_1X3X224X224_FP32_TORCH,
Expand Down
6 changes: 3 additions & 3 deletions build_tools/python/e2e_test_framework/unique_ids.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,14 @@ def hash_composite_id(keys: Sequence[str]) -> str:
MODEL_EFFICIENTNET_B7_FP32_TORCH = "68caa96e-b8bb-48a2-bb08-a3044981a370"

MODEL_BERT_LARGE_1X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-1"
MODEL_BERT_LARGE_8X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-8"
MODEL_BERT_LARGE_16X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-16"
MODEL_BERT_LARGE_24X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-24"
MODEL_BERT_LARGE_32X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-32"
MODEL_BERT_LARGE_48X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-48"
MODEL_BERT_LARGE_64X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-64"
MODEL_BERT_LARGE_128X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-128"
MODEL_BERT_LARGE_256X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-256"
MODEL_BERT_LARGE_512X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-512"
MODEL_BERT_LARGE_1024X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-1024"
MODEL_BERT_LARGE_1280X384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-1280"

MODEL_RESNET50_1X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-1"
MODEL_RESNET50_8X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-8"
Expand Down
28 changes: 19 additions & 9 deletions tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,47 +212,57 @@ iree_fetch_artifact(
NAME
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-1"
SOURCE_URL
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_1/linalg.mlir"
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_1/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-1_BertLargePTBatch1.mlir"
UNPACK
)

iree_fetch_artifact(
NAME
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-8"
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-16"
SOURCE_URL
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_8/linalg.mlir"
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_16/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-8_BertLargePTBatch8.mlir"
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-16_BertLargePTBatch16.mlir"
UNPACK
)

iree_fetch_artifact(
NAME
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-16"
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-24"
SOURCE_URL
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_16/linalg.mlir"
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_24/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-16_BertLargePTBatch16.mlir"
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-24_BertLargePTBatch24.mlir"
UNPACK
)

iree_fetch_artifact(
NAME
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-32"
SOURCE_URL
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_32/linalg.mlir"
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_32/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-32_BertLargePTBatch32.mlir"
UNPACK
)

iree_fetch_artifact(
NAME
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-48"
SOURCE_URL
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_48/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-48_BertLargePTBatch48.mlir"
UNPACK
)

iree_fetch_artifact(
NAME
"model-cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-64"
SOURCE_URL
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/BERT_LARGE/batch_64/linalg.mlir"
"https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/BERT_LARGE/batch_64/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_cbc5e400-7c93-4844-aca8-bce8f1bf9948-batch-64_BertLargePTBatch64.mlir"
UNPACK
Expand Down
Loading

0 comments on commit 92cc535

Please sign in to comment.