new dict

adamjgrant · May 6, 2024 · 6843b45 · 6843b45
1 parent 94600a7
commit 6843b45
Show file tree

Hide file tree

Showing 6 changed files with 3 additions and 3 deletions.
diff --git a/backup/dictionary.pkl b/backup/dictionary.pkl
diff --git a/backup/processing_progress.txt b/backup/processing_progress.txt
@@ -1 +1 @@
-26874314,24799999999
+28611477,26399999999
diff --git a/dictionary.msgpack b/dictionary.msgpack
diff --git a/lib/merge_batches.py b/lib/merge_batches.py
@@ -314,4 +314,4 @@ async def main():
         finish_merge()
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())
diff --git a/tokens.msgpack b/tokens.msgpack
diff --git a/train.py b/train.py
@@ -106,7 +106,7 @@ async def main(retain=False):
               gc.collect()
 
           if (word_count + 1) % (PRUNE_FREQUENCY * 100) == 0:
-              await asyncio.run(merge_batches())
+              await merge_batches()
 
   await create_batch(tree_store, TARGET_DICTIONARY_COUNT)