From 5e55b8e91e1bb6564e168c513b0e29ddcec825b3 Mon Sep 17 00:00:00 2001 From: sfc-gh-afedorov Date: Thu, 27 Aug 2020 17:13:18 -0700 Subject: [PATCH] [dc] Jamf improvements --- src/connectors/jamf_inventory.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/connectors/jamf_inventory.py b/src/connectors/jamf_inventory.py index be8299556..fe82389be 100644 --- a/src/connectors/jamf_inventory.py +++ b/src/connectors/jamf_inventory.py @@ -7,6 +7,7 @@ from connectors.utils import updated from runners.helpers import db, log +from runners.utils import groups_of CONNECTION_OPTIONS = [ @@ -22,12 +23,12 @@ ] HEADERS: dict = {} -REQUEST_SPREAD_IN_SECONDS = 180 +REQUEST_SPEED_PER_SECOND = 10 -async def fetch(session, url, fetch_over=0) -> dict: - if fetch_over: - await asyncio.sleep(fetch_over * random()) +async def fetch(session, url, wait=0) -> dict: + if wait: + await asyncio.sleep(wait) async with session.get( f'https://snowflake.jamfcloud.com/JSSResource{url}', headers=HEADERS ) as response: @@ -45,8 +46,8 @@ async def fetch(session, url, fetch_over=0) -> dict: return result -def fetch_computer(s, cid): - return fetch(s, f'/computers/id/{cid}', fetch_over=REQUEST_SPREAD_IN_SECONDS) +def fetch_computer(s, cid, i=0): + return fetch(s, f'/computers/id/{cid}', wait=i / REQUEST_SPEED_PER_SECOND) async def main(table_name): @@ -57,7 +58,7 @@ async def main(table_name): log.info(f'loading {len(cids)} computer details') computers = await asyncio.gather( - *[fetch_computer(session, cid) for cid in cids] + *[fetch_computer(session, cid, i) for i, cid in enumerate(cids)] ) log.info(f'inserting {len(computers)} computers into {table_name}') @@ -67,7 +68,8 @@ async def main(table_name): ) for cid, c in zip(cids, computers) ] - db.insert(table_name, rows) + for g in groups_of(100, rows): + db.insert(table_name, g) return len(rows)