Skip to content

Commit

Permalink
Remove multiprocessing
Browse files Browse the repository at this point in the history
Need to check better whether it reaches the end.
  • Loading branch information
nemobis committed Aug 20, 2023
1 parent 5a9f480 commit 1b5a04f
Showing 1 changed file with 4 additions and 13 deletions.
17 changes: 4 additions & 13 deletions src/prefill.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
# Distributed under the terms of the MIT license.
#

from multiprocessing import Pool, TimeoutError
import pywikibot
import sys
import random
Expand Down Expand Up @@ -43,18 +42,10 @@ def prefill_cache(max_pages=5000):
random.shuffle(sortedpages)

print(("INFO: Will start working on {} pages".format(len(sortedpages))))
# Takes almost 1 GB of RAM. With 20 processes, more than 1 CPU is needed.
with Pool(processes=10) as pool:
result = pool.map_async(worker, sortedpages, 100)
while True:
# FIXME: Doesn't respect max_pages
sleep(1)
try:
print(result.get(12000))
except TimeoutError:
print("WARNING: One chunk timed out")
except StopIteration:
print("INFO: We run out of results to print")
# FIXME: Doesn't respect max_pages
# TODO: Restore multithreading
for p in sortedpages:
worker(p)

if __name__ == '__main__':
if len(sys.argv) >= 2:
Expand Down

0 comments on commit 1b5a04f

Please sign in to comment.