adding quantizing notes for pytorch files

SolidRusT · Mar 26, 2024 · 7d11054 · 7d11054
1 parent 7cd9a24
commit 7d11054
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 5 deletions.
diff --git a/awq/NOTES.md b/awq/NOTES.md
@@ -0,0 +1,5 @@
+# QuantKit
+
+```bash
+quantkit safetensor cognitivecomputations/dolphin-2.8-gemma-2b --delete-original
+```
diff --git a/awq/requirements.txt b/awq/requirements.txt
@@ -1,8 +1,6 @@
-#autoawq==0.1.8
-#autoawq-kernels==0.0.1
 autoawq
 autoawq-kernels
 transformers
 torch
 pandas
-#exllamav2
+llm-quantkit
diff --git a/awq/run-quant-awq.py b/awq/run-quant-awq.py
@@ -8,8 +8,8 @@
 
 access_token = os.getenv('HUGGING_FACE_HUB_TOKEN')
 
-model_path = "ResplendentAI/DaturaCookie_7B"
-quant_path = 'temp/DaturaCookie_7B-AWQ'
+model_path = "temp/dolphin-2.8-gemma-2b"
+quant_path = 'temp/dolphin-2.8-gemma-2b-AWQ'
 
 quant_config = {
     "zero_point": True,