idk man

adamjgrant · Sep 26, 2024 · 7771364 · 7771364
1 parent 06781bc
commit 7771364
Show file tree

Hide file tree

Showing 5 changed files with 2 additions and 2 deletions.
diff --git a/backup/dictionary.pkl b/backup/dictionary.pkl
diff --git a/backup/processing_progress.txt b/backup/processing_progress.txt
diff --git a/dictionary.msgpack b/dictionary.msgpack
diff --git a/tokens.msgpack b/tokens.msgpack
diff --git a/train.py b/train.py
@@ -131,8 +131,8 @@ async def main(retain=False):
 
             # Silencing for now. Creating too many problems.
             # Merge batches periodically
-            # if (word_count + 1) % (PRUNE_FREQUENCY * 100) == 0:
-            #    await merge_batches()
+            if (word_count + 1) % (PRUNE_FREQUENCY * 50) == 0:
+                await merge_batches()
 
     # Final batch creation after processing is complete
     await create_batch(tree_store, TARGET_DICTIONARY_COUNT)