Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cloud Event Abstraction #13542

Merged
merged 14 commits into from
Sep 4, 2020
2 changes: 1 addition & 1 deletion sdk/eventgrid/azure-eventgrid/azure/eventgrid/_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def decode_cloud_event(self, cloud_event, **kwargs):
encode = kwargs.pop('encoding', 'utf-8')
try:
cloud_event = CloudEvent._from_json(cloud_event, encode)
deserialized_event = CloudEvent.deserialize(cloud_event)
deserialized_event = CloudEvent._from_generated(cloud_event)
CloudEvent._deserialize_data(deserialized_event, deserialized_event.type)
return deserialized_event
except Exception as err:
Expand Down
82 changes: 54 additions & 28 deletions sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from msrest.serialization import UTC
import datetime as dt
import uuid
import base64
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not needed anymore

rakshith91 marked this conversation as resolved.
Show resolved Hide resolved
import json
import six
from ._generated import models
Expand Down Expand Up @@ -50,7 +51,7 @@ def _from_json(event, encode):
return event


class CloudEvent(InternalCloudEvent, EventMixin): #pylint:disable=too-many-instance-attributes
class CloudEvent(EventMixin): #pylint:disable=too-many-instance-attributes
"""Properties of an event published to an Event Grid topic using the CloudEvent 1.0 Schema.

All required parameters must be populated in order to send to Azure.
Expand All @@ -75,35 +76,60 @@ class CloudEvent(InternalCloudEvent, EventMixin): #pylint:disable=too-many-ins
unique for each distinct event.
:type id: Optional[str]
"""

_validation = {
'source': {'required': True},
'type': {'required': True},
}

_attribute_map = {
'additional_properties': {'key': '', 'type': '{object}'},
'id': {'key': 'id', 'type': 'str'},
'source': {'key': 'source', 'type': 'str'},
'data': {'key': 'data', 'type': 'object'},
'data_base64': {'key': 'data_base64', 'type': 'bytearray'},
'type': {'key': 'type', 'type': 'str'},
'time': {'key': 'time', 'type': 'iso-8601'},
'specversion': {'key': 'specversion', 'type': 'str'},
'dataschema': {'key': 'dataschema', 'type': 'str'},
'datacontenttype': {'key': 'datacontenttype', 'type': 'str'},
'subject': {'key': 'subject', 'type': 'str'},
}

def __init__(self, source, type, **kwargs):
# type: (str, str, Any) -> None
kwargs.setdefault('id', uuid.uuid4())
kwargs.setdefault("source", source)
kwargs.setdefault("type", type)
kwargs.setdefault("time", dt.datetime.now(UTC()).isoformat())
kwargs.setdefault("specversion", "1.0")

super(CloudEvent, self).__init__(**kwargs)
self.source = source
self.type = type
self.specversion = kwargs.pop("specversion", "1.0")
self.id = kwargs.pop("id", str(uuid.uuid4()))
self.time = kwargs.pop("time", dt.datetime.now(UTC()).isoformat())
self.data = kwargs.pop("data", None)
self.datacontenttype = kwargs.pop("datacontenttype", None)
self.dataschema = kwargs.pop("dataschema", None)
self.subject = kwargs.pop("subject", None)
self._extensions = {}
self._extensions.update({k:v for k, v in kwargs.pop('extensions', {}).items()})
Copy link
Member

@lmazuel lmazuel Sep 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How is extensions available to people if we make it private? They should be able to read from it


@classmethod
def _from_generated(cls, cloud_event, **kwargs):
generated = InternalCloudEvent.deserialize(cloud_event)
if generated.additional_properties:
extensions = {k:v for k, v in generated.additional_properties.items()}
kwargs.setdefault('extensions', extensions)
return cls(
id=generated.id,
source=generated.source,
type=generated.type,
specversion=generated.specversion,
data=generated.data or generated.data_base64,
time=generated.time,
dataschema=generated.dataschema,
datacontenttype=generated.datacontenttype,
subject=generated.subject,
**kwargs
)

def _to_generated(self, **kwargs):
if isinstance(self.data, six.binary_type):
data_base64 = self.data
data = None
else:
data_base64 = None
data = self.data
return InternalCloudEvent(
id=self.id,
source=self.source,
type=self.type,
specversion=self.specversion,
data=data,
data_base64=data_base64,
time=self.time,
dataschema=self.dataschema,
datacontenttype=self.datacontenttype,
subject=self.subject,
additional_properties=self._extensions,
**kwargs
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Extensions should be passed part of additional_properties

In [3]: c=CloudEvent(id=42, source="source", type="type", specversion="1.0", additional_properties={"foo":True})

In [4]: c.serialize()
Out[4]:
{'foo': True,
 'id': '42',
 'source': 'source',
 'type': 'type',
 'specversion': '1.0'}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apologies if I misremembered our call the other day but did we reach a conclusion about not using kwargs for extensions? My fear being that we've just lost symmetry if someone **'s a serialized cloudevent back into itself.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i took the liberty to slightly change the behavior to this

CloudEvent(id=42, source="source", type="type", specversion="1.0", ext_as_kw = "hello",  extensions={"foo":True})

will serialize to

{'foo': True,
'ext_as_kw ': 'hello',
 'id': '42',
 'source': 'source',
 'type': 'type',
 'specversion': '1.0'}

In other words, extensions can be accepted as both kwargs and an explicit extensions dictionary

Rationale is

  1. This behavior is quite similar to the CloudEvent sdk's model which i think would benefit users jumping from there.
  2. additional_properties as such can be sent by kwargs when possible

I can disallow kwargs if there is a strong opinion against this

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The pitfall here is if someone has an additional property called additional_properties, we get "undefined behavior". I seem to recall a mention in the call that we weren't TOO worried about that/could document it/reserved keywords and all that, but it's the one thing that irks me.

This may be a good target for getting some broader user input on, having them hit this pitfall and see if it's dig-out-able or understandable to their eyes.

Copy link
Member

@lmazuel lmazuel Sep 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There should be one-- and preferably only one --obvious way to do it.

:)

What if you do CloudEvent(foo=True, extensions={"foo":False}) for instance? I think we are opening a pandora's box of things we're unsure the day before the code complete. I would suggest we keep extensions only but make this a priority question in user study.

I'm not against it and it has appeal, but right now I want to be sure we have the base minimal behavior, and improve from it first.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changed it to accepting only extensions - thank you both for the inputs

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also @KieranBrantnerMagee , I'm catching up your comment, additional_properties is just for autorest model, I was explaining how to tell autorest model to serialize attributes that were not on the Swagger. It was not a comment about the external API of public CloudEvent for customer.

)


class EventGridEvent(InternalEventGridEvent, EventMixin):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ def send(self, events, **kwargs):
events = [events]

if all(isinstance(e, CloudEvent) for e in events) or all(_is_cloud_event(e) for e in events):
try:
events = [e._to_generated(**kwargs) for e in events]
except AttributeError:
pass # means it's a dictionary
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rakshith91 still, how could this happen?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorry - missed it
dictionary will not have a _to_generated - so we do nothing and assume it's in the right format

kwargs.setdefault("content_type", "application/cloudevents-batch+json; charset=utf-8")
self._client.publish_cloud_event_events(self._topic_hostname, events, **kwargs)
elif all(isinstance(e, EventGridEvent) for e in events) or all(isinstance(e, dict) for e in events):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ async def send(self, events, **kwargs):
events = [events]

if all(isinstance(e, CloudEvent) for e in events) or all(_is_cloud_event(e) for e in events):
try:
events = [e._to_generated(**kwargs) for e in events]
except AttributeError:
pass # means it's a dictionary
kwargs.setdefault("content_type", "application/cloudevents-batch+json; charset=utf-8")
await self._client.publish_cloud_event_events(self._topic_hostname, events, **kwargs)
elif all(isinstance(e, EventGridEvent) for e in events) or all(isinstance(e, dict) for e in events):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"id": "3dc4b913-4bc2-41f8-be9b-bf1f67069806", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:41.947462Z",
body: '[{"id": "4e7a9df5-8e28-42c8-adca-4412c93e92d5", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:13.810619Z",
"specversion": "1.0"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:43 GMT
- Thu, 03 Sep 2020 21:37:12 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"id": "51c18497-2a25-45f1-b9ba-fdaf08c00263", "source": "http://samplesource.dev",
"data": {"sample": "cloudevent"}, "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:42.304479Z",
body: '[{"id": "51c747e0-7e8f-467c-80e4-920fdf1d95d2", "source": "http://samplesource.dev",
"data": {"sample": "cloudevent"}, "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:14.383108Z",
"specversion": "1.0"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:43 GMT
- Thu, 03 Sep 2020 21:37:12 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"id": "6a315e93-a59c-4eca-b2f2-6bf3b8f27984", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:42.629467Z",
body: '[{"id": "eef0ee57-9833-44fd-9f52-d5f20b74267d", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:14.803665Z",
"specversion": "1.0"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:43 GMT
- Thu, 03 Sep 2020 21:37:13 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
interactions:
- request:
body: '[{"reason_code": 204, "extension": "extension", "id": "9da8e4ce-df2c-4480-b809-c7132fb3ee74",
"source": "http://samplesource.dev", "data": "cloudevent", "type": "Sample.Cloud.Event",
"time": "2020-09-03T21:37:15.165652Z", "specversion": "1.0"}]'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '244'
Content-Type:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
body:
string: ''
headers:
api-supported-versions:
- '2018-01-01'
content-length:
- '0'
date:
- Thu, 03 Sep 2020 21:37:13 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
- max-age=31536000; includeSubDomains
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -28,7 +26,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:44 GMT
- Thu, 03 Sep 2020 21:37:13 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"customSubject": "sample", "customEventType": "sample.event", "customDataVersion":
"2.0", "customId": "1234", "customEventTime": "2020-08-19T03:36:56.936961+00:00",
"2.0", "customId": "1234", "customEventTime": "2020-09-03T21:37:32.440554+00:00",
"customData": "sample data"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/json
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- uPQPJHQHsAhBxWOWtRXslz3sXf7TJ5lcqLZ6SC4QzJ4=
method: POST
uri: https://customeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:57 GMT
- Thu, 03 Sep 2020 21:37:31 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
interactions:
- request:
body: '[{"customSubject": "sample", "customEventType": "sample.event", "customDataVersion":
"2.0", "customId": "1234", "customEventTime": "2020-08-19T03:36:57.422734+00:00",
"2.0", "customId": "1234", "customEventTime": "2020-09-03T21:37:32.857620+00:00",
"customData": "sample data"}, {"customSubject": "sample2", "customEventType":
"sample.event", "customDataVersion": "2.0", "customId": "12345", "customEventTime":
"2020-08-19T03:36:57.422734+00:00", "customData": "sample data 2"}]'
"2020-09-03T21:37:32.857620+00:00", "customData": "sample data 2"}]'
headers:
Accept:
- '*/*'
Expand All @@ -18,8 +18,6 @@ interactions:
- application/json
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- uPQPJHQHsAhBxWOWtRXslz3sXf7TJ5lcqLZ6SC4QzJ4=
method: POST
uri: https://customeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -28,12 +26,10 @@ interactions:
headers:
api-supported-versions:
- '2018-01-01'
connection:
- close
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:58 GMT
- Thu, 03 Sep 2020 21:37:31 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
interactions:
- request:
body: '[{"id": "576f3e27-5433-47b3-8b4e-e31ea6a1bdde", "subject": "sample", "data":
"eventgridevent", "eventType": "Sample.EventGrid.Event", "eventTime": "2020-08-19T03:37:11.07064Z",
"dataVersion": "2.0"}, {"id": "6c359c8d-b1fe-4458-96c5-eaffcd864573", "subject":
body: '[{"id": "a3ad167e-302d-4d3d-b5ee-d9e989aa277a", "subject": "sample", "data":
"eventgridevent", "eventType": "Sample.EventGrid.Event", "eventTime": "2020-09-03T21:37:46.777197Z",
"dataVersion": "2.0"}, {"id": "e36112d1-2413-462b-ac5a-8046bd7add59", "subject":
"sample2", "data": "eventgridevent2", "eventType": "Sample.EventGrid.Event",
"eventTime": "2020-08-19T03:37:11.071641Z", "dataVersion": "2.0"}]'
"eventTime": "2020-09-03T21:37:46.778197Z", "dataVersion": "2.0"}]'
headers:
Accept:
- '*/*'
Expand All @@ -13,13 +13,11 @@ interactions:
Connection:
- keep-alive
Content-Length:
- '401'
- '402'
Content-Type:
- application/json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dS38eoc9IPjofR5npZ1QXrsb3Gz/Kdt99ZdK9SJ+99w=
method: POST
uri: https://eventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -31,7 +29,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:37:12 GMT
- Thu, 03 Sep 2020 21:37:45 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Loading