Update benchmarks.sh

pytorch · Oct 4, 2024 · 2bb4c89 · 2bb4c89
1 parent b16772d
commit 2bb4c89
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 3 deletions.
diff --git a/torchao/_models/llama/benchmarks.sh b/torchao/_models/llama/benchmarks.sh
@@ -21,6 +21,7 @@ export MODEL_REPO=meta-llama/Meta-Llama-3.1-8B
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization int8wo --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization int4wo-64 --write_result benchmark_results.txt
+# Runs on H100, float8 is not supported on CUDA arch < 8.9
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization float8wo --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization float8dq-tensor --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization float8dq-wo --write_result benchmark_results.txt

diff --git a/torchao/quantization/README.md b/torchao/quantization/README.md
@@ -139,9 +139,6 @@ change_linear_weights_to_int8_dqtensors(model)
 from torchao.quantization import quantize_, float8_dynamic_activation_float8_weight
 from torchao.quantization.observer import PerTensor
 quantize_(model, float8_dynamic_activation_float8_weight(granularity=PerTensor()))
-from torchao.quantization.observer import PerTensor
-quantize_(model, float8_dynamic_activation_float8_weight(granularity=PerTensor()))
-
 ```
 
 #### A16W6 Floating Point WeightOnly Quantization