-
Notifications
You must be signed in to change notification settings - Fork 57
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[DC] Meraki Device Connector #290
Changes from 55 commits
c001f5e
1d8a45b
b0e4801
b30174b
24f0664
92b01f0
179325b
f509053
d777a21
706624a
81f4a11
327ae3f
fd457c3
fcc2520
2a8c475
bb027a5
d0eb88a
a881d9b
e413652
d7bb036
47859d9
db03993
e7487e1
b154502
d39979e
f78dc8a
4bf02cb
a0f99f4
409b2d0
f7d259d
801d6cc
64276bc
54ef870
279338c
fe2fb8f
e7be0aa
f903a47
a9f25dd
dc91f32
eef66e7
09bffa6
26d1589
2ff4139
6260259
9d8b261
af7c6c4
a386650
c020007
75e301e
a8f2f38
7ae202a
a0fb70c
c09881b
383b997
b466ccc
b147f05
158ccd8
78acd49
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,188 @@ | ||
"""Meraki Devices | ||
Collect Meraki Device information using an API Token | ||
""" | ||
|
||
from runners.helpers import log | ||
from runners.helpers import db | ||
from runners.helpers.dbconfig import ROLE as SA_ROLE | ||
|
||
from datetime import datetime | ||
|
||
import requests | ||
from urllib.error import HTTPError | ||
from .utils import yaml_dump | ||
|
||
|
||
PAGE_SIZE = 5 | ||
|
||
CONNECTION_OPTIONS = [ | ||
{ | ||
'name': 'api_token', | ||
'title': "Meraki API Token", | ||
'prompt': "Your Meraki API Token", | ||
'type': 'str', | ||
'secret': True, | ||
'required': True, | ||
}, | ||
{ | ||
'name': 'network_id_whitelist', | ||
'title': "Meraki Network Ids Whitelist", | ||
'prompt': "Whitelist of Network Ids", | ||
'type': 'list', | ||
'secret': False, | ||
'required': True, | ||
}, | ||
] | ||
|
||
LANDING_TABLE_COLUMNS_CLIENT = [ | ||
('INSERT_ID', 'NUMBER IDENTITY START 1 INCREMENT 1'), | ||
('SNAPSHOT_AT', 'TIMESTAMP_LTZ(9)'), | ||
('RAW', 'VARIANT'), | ||
('ID', 'VARCHAR(256)'), | ||
('MAC', 'VARCHAR(256)'), | ||
('DESCRIPTION', 'VARCHAR(256)'), | ||
('MDNS_NAME', 'VARCHAR(256)'), | ||
('DHCP_HOSTNAME', 'VARCHAR(256)'), | ||
('IP', 'VARCHAR(256)'), | ||
('SWITCHPORT', 'VARCHAR(256)'), | ||
('VLAN', 'INT'), | ||
('USAGE_SENT', 'INT'), | ||
('USAGE_RECV', 'INT'), | ||
('SERIAL', 'VARCHAR(256)'), | ||
] | ||
|
||
LANDING_TABLE_COLUMNS_DEVICE = [ | ||
('INSERT_ID', 'NUMBER IDENTITY START 1 INCREMENT 1'), | ||
('SNAPSHOT_AT', 'TIMESTAMP_LTZ(9)'), | ||
('RAW', 'VARIANT'), | ||
('SERIAL', 'VARCHAR(256)'), | ||
('ADDRESS', 'VARCHAR(256)'), | ||
('NAME', 'VARCHAR(256)'), | ||
('NETWORK_ID', 'VARCHAR(256)'), | ||
('MODEL', 'VARCHAR(256)'), | ||
('MAC', 'VARCHAR(256)'), | ||
('LAN_IP', 'VARCHAR(256)'), | ||
('WAN_1_IP', 'VARCHAR(256)'), | ||
('WAN_2_IP', 'VARCHAR(256)'), | ||
('TAGS', 'VARCHAR(256)'), | ||
('LNG', 'FLOAT'), | ||
('LAT', 'FLOAT'), | ||
] | ||
|
||
|
||
def get_data(url: str, token: str, params: dict = {}) -> dict: | ||
headers: dict = { | ||
"Content-Type": "application/json", | ||
"Accept": "application/json", | ||
"X-Cisco-Meraki-API-Key": f"{token}", | ||
} | ||
try: | ||
log.debug(f"Preparing GET: url={url} with params={params}") | ||
req = requests.get(url, params=params, headers=headers) | ||
req.raise_for_status() | ||
except HTTPError as http_err: | ||
log.error(f"Error GET: url={url}") | ||
log.error(f"HTTP error occurred: {http_err}") | ||
raise | ||
log.debug(req.status_code) | ||
return req.json() | ||
|
||
|
||
def connect(connection_name, options): | ||
landing_table_client = f'data.meraki_devices_{connection_name}_client_connection' | ||
landing_table_device = f'data.meraki_devices_{connection_name}_device_connection' | ||
options['network_id_whitelist'] = options.get('network_id_whitelist', '').split(',') | ||
|
||
comment = yaml_dump(module='meraki_devices', **options) | ||
|
||
db.create_table(name=landing_table_client, | ||
cols=LANDING_TABLE_COLUMNS_CLIENT, | ||
comment=comment) | ||
db.execute(f'GRANT INSERT, SELECT ON {landing_table_client} TO ROLE {SA_ROLE}') | ||
|
||
db.create_table(name=landing_table_device, | ||
cols=LANDING_TABLE_COLUMNS_DEVICE, | ||
comment=comment) | ||
db.execute(f'GRANT INSERT, SELECT ON {landing_table_device} TO ROLE {SA_ROLE}') | ||
return { | ||
'newStage': 'finalized', | ||
'newMessage': "Meraki ingestion tables created!", | ||
} | ||
|
||
|
||
def ingest(table_name, options): | ||
ingest_type = 'client' if table_name.endswith('_CLIENT_CONNECTION') else 'device' | ||
print("ingest_type: ", ingest_type) | ||
print("table_name: ", table_name) | ||
landing_table = f'data.{table_name}' | ||
|
||
timestamp = datetime.utcnow() | ||
api_key = options['api_key'] | ||
whitelist = options['network_id_whitelist'] | ||
|
||
for network in whitelist: | ||
try: | ||
devices = get_data(f"https://api.meraki.com/api/v0/networks/{network}/devices", api_key) | ||
except requests.exceptions.HTTPError as e: | ||
log.error(f"{network} not accessible, ") | ||
log.error(e) | ||
continue | ||
|
||
if ingest_type == 'device': | ||
db.insert( | ||
landing_table, | ||
values=[( | ||
timestamp, | ||
device, | ||
device.get('serial'), | ||
device.get('address'), | ||
device.get('name'), | ||
device.get('networkId'), | ||
device.get('model'), | ||
device.get('mac'), | ||
device.get('lanIp'), | ||
device.get('wan1Ip'), | ||
device.get('wan2Ip'), | ||
device.get('tags'), | ||
device.get('lng'), | ||
device.get('lat'), | ||
) for device in devices], | ||
select=db.derive_insert_select(LANDING_TABLE_COLUMNS_DEVICE), | ||
columns=db.derive_insert_columns(LANDING_TABLE_COLUMNS_DEVICE) | ||
) | ||
log.info(f'Inserted {len(devices)} rows ({landing_table}).') | ||
yield len(devices) | ||
|
||
else: | ||
for device in devices: | ||
serial_number = device['serial'] | ||
|
||
try: | ||
clients = get_data(f"https://api.meraki.com/api/v0/devices/{serial_number}/clients", api_key) | ||
except requests.exceptions.HTTPError as e: | ||
log.error(f"{network} not accessible, ") | ||
log.error(e) | ||
continue | ||
|
||
db.insert( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sorry I wasn't more clear on this. could you please replace this db.insert expression with --
I think it will do the same thing but more succinctly. to clarify There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. also, are you sure There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sometimes There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Got it, interesting. If you're sure that the others never return
might help to future folks, as well There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've never seen that occur with usage:recv or usage:sent, but I think it'll be best if we used the same approach for those instances too. Thank you for this feedback! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sgtm |
||
landing_table, | ||
values=[( | ||
timestamp, | ||
client, | ||
client.get('id'), | ||
client.get('mac'), | ||
client.get('description'), | ||
client.get('mdnsName'), | ||
client.get('dhcpHostname'), | ||
client.get('ip'), | ||
client.get('switchport'), | ||
client.get('vlan') or None, # vlan sometimes set to '' | ||
client.get('usage', {}).get('sent') or None, | ||
client.get('usage', {}).get('recv') or None, | ||
serial_number, | ||
) for client in clients], | ||
select=db.derive_insert_select(LANDING_TABLE_COLUMNS_CLIENT), | ||
columns=db.derive_insert_columns(LANDING_TABLE_COLUMNS_CLIENT) | ||
) | ||
log.info(f'Inserted {len(clients)} rows ({landing_table}).') | ||
yield len(clients) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this will return a
Union[str, list]
when I think you'd be better off with just alist
, i.e. put[]
in the second param ofget
.also, up to you if you're comfortable, but I would have made this a change to
connectors_runner.py
and added thelist
type to be parsed there next tojson
. later on, we can also add a special UI element.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I guess I have a few questions. This function "returns" a dictionary with a message but it creates a table and grants roles. What are you referring to when you say "This will return a
Union[str, list]
"? Are you referring tooptions
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Would adding the list type in connectors_runner.py parse the user input as a list? When we tested this particular input, we got it as a string, like
'id1, id2, id3'
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How does this look? @andrey-snowflake
@greg-snowflake #306
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perfect, thanks.
Err, had a bit of a think-o. The "return" type I was thinking of was of the expression
but since it splits the empty string, it's always a list.