Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add new streams #3

Merged
merged 4 commits into from
Mar 18, 2021
Merged

Add new streams #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6fd362b
Add custom fields and location methods to client
stkbailey Mar 18, 2021
d333da4
Add custom fields tap
stkbailey Mar 18, 2021
2a14312
Add custom field values and location streams
stkbailey Mar 18, 2021
97d37de
Fix request params issue
stkbailey Mar 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions tap_zenefits/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@ def __init__(self, api_key):
self._client = requests.Session()
self._client.headers.update({'Authorization': f'Bearer {api_key}'})

def fetch_custom_fields(self, starting_after=None):
url = f"{self.BASE_URL}/core/custom_fields"
return self._client.get(url, params=None).json()

def fetch_custom_field_values(self, starting_after=None):
url = f"{self.BASE_URL}/core/custom_field_values"
params = { "starting_after": starting_after } if starting_after else None
return self._client.get(url, params=params).json()

def fetch_departments(self, company_id=None, starting_after=None):
if self.USE_COMPANY_ID:
url = f"{self.BASE_URL}/core/companies/{company_id}/departments"
Expand All @@ -22,6 +31,11 @@ def fetch_employments(self, starting_after=None):
params = { "starting_after": starting_after } if starting_after else None
return self._client.get(url, params=params).json()

def fetch_locations(self, starting_after=None):
url = f"{self.BASE_URL}/core/locations"
params = { "starting_after": starting_after } if starting_after else None
return self._client.get(url, params=params).json()

def fetch_people(self, company_id=None, starting_after=None):
if self.USE_COMPANY_ID:
url = f"{self.BASE_URL}/core/companies/{company_id}/people"
Expand Down
11 changes: 10 additions & 1 deletion tap_zenefits/discover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@

from .streams import STREAMS

DEFAULT_SELECTED_STREAMS = [
"custom_fields",
"custom_field_values",
"departments",
"employments",
"locations",
"people",
]

def get_abs_path(path):
return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)

Expand Down Expand Up @@ -33,7 +42,7 @@ def get_schemas():
meta = metadata.write(meta, ('properties', stream_object.replication_key), 'inclusion', 'automatic')

# Select a few default-selected strings
if stream_name in ["people", "departments", "employments"]:
if stream_name in DEFAULT_SELECTED_STREAMS:
meta = metadata.write(meta, (), 'selected', "true")

meta = metadata.to_list(meta)
Expand Down
48 changes: 48 additions & 0 deletions tap_zenefits/schemas/custom_field_values.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
{
"type": "object",
"properties": {
"url": {
"type": "string"
},
"object": {
"type": "string"
},
"custom_field": {
"type": "object",
"properties": {
"url": {
"type": "string"
},
"object": {
"type": "string"
},
"ref_object": {
"type": "string"
}
},
"additionalProperties": true
},
"value": {
"type": ["string", "null"]
},
"person": {
"type": "object",
"properties": {
"url": {
"type": "string"
},
"object": {
"type": "string"
},
"ref_object": {
"type": "string"
}
},
"additionalProperties": true
},
"id": {
"type": "string"
}
},
"additionalProperties": true
}
72 changes: 72 additions & 0 deletions tap_zenefits/schemas/custom_fields.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
{
"type": "object",
"properties": {
"can_manager_view_field": {
"type": "boolean"
},
"custom_field_values": {
"type": "object",
"properties": {
"url": {
"type": "string"
},
"object": {
"type": "string"
},
"ref_object": {
"type": "string"
}
},
"additionalProperties": true
},
"person_during_onboarding": {
"type": "boolean"
},
"name": {
"type": "string"
},
"is_field_completer_person": {
"type": "boolean"
},
"url": {
"type": "string"
},
"is_sensitive": {
"type": "boolean"
},
"is_field_required": {
"type": "boolean"
},
"object": {
"type": "string"
},
"can_person_view_field": {
"type": "boolean"
},
"can_person_edit_field": {
"type": "boolean"
},
"company_during_hiring": {
"type": "boolean"
},
"custom_field_type": {
"type": "string"
},
"help_url": {
"type": ["string", "null"]
},
"help_text": {
"type": "string"
},
"help_url_media": {
"type": ["string", "null"]
},
"id": {
"type": "string"
},
"media_file_type": {
"type": ["string", "null"]
}
},
"additionalProperties": true
}
69 changes: 69 additions & 0 deletions tap_zenefits/schemas/locations.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"type": "object",
"properties": {
"city": {
"type": ["string", "null"]
},
"name": {
"type": ["string", "null"]
},
"zip": {
"type": ["string", "null"]
},
"people": {
"type": "object",
"properties": {
"url": {
"type": ["string", "null"]
},
"object": {
"type": ["string", "null"]
},
"ref_object": {
"type": ["string", "null"]
}
},
"additionalProperties": true
},
"url": {
"type": ["string", "null"]
},
"street1": {
"type": ["string", "null"]
},
"street2": {
"type": ["string", "null"]
},
"object": {
"type": ["string", "null"]
},
"phone": {
"type": ["string", "null"]
},
"state": {
"type": ["string", "null"]
},
"country": {
"type": ["string", "null"]
},
"company": {
"type": "object",
"properties": {
"url": {
"type": "string"
},
"object": {
"type": "string"
},
"ref_object": {
"type": "string"
}
},
"additionalProperties": true
},
"id": {
"type": "string"
}
},
"additionalProperties": true
}
57 changes: 55 additions & 2 deletions tap_zenefits/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,39 @@ class CatalogStream(Stream):
class FullTableStream(Stream):
replication_method = 'FULL_TABLE'


class CustomFields(FullTableStream):
tap_stream_id = 'custom_fields'
key_properties = ['id']
object_type = 'CUSTOM_FIELDS'

def sync(self, *args, **kwargs):
next_url = "True"
while next_url:
starting_after = get_starting_after(next_url)
response = self.client.fetch_custom_fields(starting_after)
data = response.get('data', {})
values = data.get('data', [])
for record in values:
yield record
next_url = data.get('next_url', None)

class CustomFieldValues(FullTableStream):
tap_stream_id = 'custom_field_values'
key_properties = ['id']
object_type = 'CUSTOM_FIELD_VALUES'

def sync(self, *args, **kwargs):
next_url = "True"
while next_url:
starting_after = get_starting_after(next_url)
response = self.client.fetch_custom_field_values(starting_after)
data = response.get('data', {})
values = data.get('data', [])
for record in values:
yield record
next_url = data.get('next_url', None)

class Departments(FullTableStream):
tap_stream_id = 'departments'
key_properties = ['id']
Expand All @@ -48,7 +81,7 @@ def sync(self, company_id=None, *args, **kwargs):
for department in departments:
yield department
next_url = data.get('next_url', None)

class Employments(FullTableStream):
tap_stream_id = 'employments'
key_properties = ['id']
Expand All @@ -65,6 +98,22 @@ def sync(self, *args, **kwargs):
yield employment
next_url = data.get('next_url', None)

class Locations(FullTableStream):
tap_stream_id = 'locations'
key_properties = ['id']
object_type = 'LOCATIONS'

def sync(self, *args, **kwargs):
next_url = "True"
while next_url:
starting_after = get_starting_after(next_url)
response = self.client.fetch_locations(starting_after)
data = response.get('data', {})
values = data.get('data', [])
for record in values:
yield record
next_url = data.get('next_url', None)

class PayStubs(FullTableStream):
tap_stream_id = 'pay_stubs'
key_properties = ['id']
Expand Down Expand Up @@ -97,7 +146,7 @@ def sync(self, *args, **kwargs):
yield payrun
next_url = data.get('next_url', None)

class People(FullTableStream):
class People(CatalogStream):
tap_stream_id = 'people'
key_properties = ['id']
object_type = 'PEOPLE'
Expand Down Expand Up @@ -131,9 +180,13 @@ def sync(self, *args, **kwargs):
yield time_duration
next_url = data.get('next_url', None)


STREAMS = {
'custom_fields': CustomFields,
'custom_field_values': CustomFieldValues,
'departments': Departments,
'employments': Employments,
'locations': Locations,
'pay_stubs': PayStubs,
'payruns': Payruns,
'people': People,
Expand Down