From adc9e5f0194b424e9fd19e92fe66d8c6d426dc29 Mon Sep 17 00:00:00 2001 From: David Simon Date: Tue, 19 Dec 2023 09:11:16 -0500 Subject: [PATCH 1/4] Add remove_details option --- vdirsyncer/cli/config.py | 3 +++ vdirsyncer/cli/tasks.py | 1 + vdirsyncer/sync/__init__.py | 9 ++++++--- vdirsyncer/vobject.py | 35 ++++++++++++++++++++++++++++++++--- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/vdirsyncer/cli/config.py b/vdirsyncer/cli/config.py index 8aed8249..939e9d0e 100644 --- a/vdirsyncer/cli/config.py +++ b/vdirsyncer/cli/config.py @@ -237,6 +237,9 @@ def __init__(self, full_config: Config, name: str, options: dict[str, str]): options.pop("conflict_resolution", None) ) + self.required_attendee = options.pop("required_attendee", None) + self.remove_details = options.pop("remove_details", False) + try: self.collections = options.pop("collections") except KeyError: diff --git a/vdirsyncer/cli/tasks.py b/vdirsyncer/cli/tasks.py index 6e8f5ba3..ecc2afc1 100644 --- a/vdirsyncer/cli/tasks.py +++ b/vdirsyncer/cli/tasks.py @@ -79,6 +79,7 @@ def error_callback(e): force_delete=force_delete, error_callback=error_callback, partial_sync=pair.partial_sync, + remove_details=pair.remove_details, ) if sync_failed: diff --git a/vdirsyncer/sync/__init__.py b/vdirsyncer/sync/__init__.py index 25b46a32..6768c13c 100644 --- a/vdirsyncer/sync/__init__.py +++ b/vdirsyncer/sync/__init__.py @@ -42,7 +42,7 @@ def __init__(self, storage: Storage, status: SubStatus): self.status = status self._item_cache = {} # type: ignore[var-annotated] - async def prepare_new_status(self) -> bool: + async def prepare_new_status(self, remove_details: bool = False) -> bool: storage_nonempty = False prefetch = [] @@ -67,6 +67,8 @@ def _store_props(ident: str, props: ItemMetadata) -> None: # Prefetch items if prefetch: async for href, item, etag in self.storage.get_multi(prefetch): + if remove_details: + item = item.without_details() _store_props( item.ident, ItemMetadata(href=href, hash=item.hash, etag=etag), @@ -105,6 +107,7 @@ async def sync( force_delete=False, error_callback=None, partial_sync="revert", + remove_details: bool=False, ) -> None: """Synchronizes two storages. @@ -146,8 +149,8 @@ async def sync( a_info = _StorageInfo(storage_a, SubStatus(status, "a")) b_info = _StorageInfo(storage_b, SubStatus(status, "b")) - a_nonempty = await a_info.prepare_new_status() - b_nonempty = await b_info.prepare_new_status() + a_nonempty = await a_info.prepare_new_status(remove_details=remove_details) + b_nonempty = await b_info.prepare_new_status(remove_details=remove_details) if status_nonempty and not force_delete: if a_nonempty and not b_nonempty: diff --git a/vdirsyncer/vobject.py b/vdirsyncer/vobject.py index 51482215..a5e30ec4 100644 --- a/vdirsyncer/vobject.py +++ b/vdirsyncer/vobject.py @@ -57,6 +57,35 @@ def with_uid(self, new_uid): return Item("\r\n".join(parsed.dump_lines())) + def without_details(self): + """Returns a minimal version of this item. + + Filters out data to reduce content size and hide private details: + * Description + * Location + * Organizer + * Attendees list + * Redundant timezone data (actual timezone of event is preserved) + """ + parsed = _Component.parse(self.raw) + stack = [parsed] + while stack: + component = stack.pop() + + component.subcomponents = [ + subcomp for subcomp + in component.subcomponents + if subcomp.name != "VTIMEZONE" + ] + for field in ["DESCRIPTION", "ORGANIZER", "ATTENDEE", "LOCATION"]: + # Repeatedly delete because some fields can appear multiple times + while field in component: + del component[field] + + stack.extend(component.subcomponents) + + return Item("\r\n".join(parsed.dump_lines())) + @cached_property def raw(self): """Raw content of the item, as unicode string. @@ -240,9 +269,9 @@ class _Component: Raw outline of the components. Vdirsyncer's operations on iCalendar and VCard objects are limited to - retrieving the UID and splitting larger files into items. Consequently this - parser is very lazy, with the downside that manipulation of item properties - are extremely costly. + retrieving the UID, removing fields, and splitting larger files into items. + Consequently this parser is very lazy, with the downside that manipulation + of item properties are extremely costly. Other features: From b8475e47b064c2ed656708af9dc76fc61dcf7775 Mon Sep 17 00:00:00 2001 From: David Simon Date: Tue, 19 Dec 2023 19:57:11 -0500 Subject: [PATCH 2/4] Add required_attendee --- vdirsyncer/cli/tasks.py | 1 + vdirsyncer/sync/__init__.py | 17 ++++++-- vdirsyncer/vobject.py | 80 +++++++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/vdirsyncer/cli/tasks.py b/vdirsyncer/cli/tasks.py index ecc2afc1..277a9dfd 100644 --- a/vdirsyncer/cli/tasks.py +++ b/vdirsyncer/cli/tasks.py @@ -80,6 +80,7 @@ def error_callback(e): error_callback=error_callback, partial_sync=pair.partial_sync, remove_details=pair.remove_details, + required_attendee=pair.required_attendee, ) if sync_failed: diff --git a/vdirsyncer/sync/__init__.py b/vdirsyncer/sync/__init__.py index 6768c13c..fa9df3d2 100644 --- a/vdirsyncer/sync/__init__.py +++ b/vdirsyncer/sync/__init__.py @@ -42,7 +42,7 @@ def __init__(self, storage: Storage, status: SubStatus): self.status = status self._item_cache = {} # type: ignore[var-annotated] - async def prepare_new_status(self, remove_details: bool = False) -> bool: + async def prepare_new_status(self, remove_details: bool = False, required_attendee: str | None = None) -> bool: storage_nonempty = False prefetch = [] @@ -67,6 +67,8 @@ def _store_props(ident: str, props: ItemMetadata) -> None: # Prefetch items if prefetch: async for href, item, etag in self.storage.get_multi(prefetch): + if required_attendee and not item.has_confirmed_attendee(required_attendee): + continue if remove_details: item = item.without_details() _store_props( @@ -107,7 +109,8 @@ async def sync( force_delete=False, error_callback=None, partial_sync="revert", - remove_details: bool=False, + remove_details: bool = False, + required_attendee: str | None = None, ) -> None: """Synchronizes two storages. @@ -149,8 +152,14 @@ async def sync( a_info = _StorageInfo(storage_a, SubStatus(status, "a")) b_info = _StorageInfo(storage_b, SubStatus(status, "b")) - a_nonempty = await a_info.prepare_new_status(remove_details=remove_details) - b_nonempty = await b_info.prepare_new_status(remove_details=remove_details) + a_nonempty = await a_info.prepare_new_status( + remove_details=remove_details, + required_attendee=required_attendee + ) + b_nonempty = await b_info.prepare_new_status( + remove_details=remove_details, + required_attendee=required_attendee + ) if status_nonempty and not force_delete: if a_nonempty and not b_nonempty: diff --git a/vdirsyncer/vobject.py b/vdirsyncer/vobject.py index a5e30ec4..aadef784 100644 --- a/vdirsyncer/vobject.py +++ b/vdirsyncer/vobject.py @@ -56,6 +56,20 @@ def with_uid(self, new_uid): component["UID"] = new_uid return Item("\r\n".join(parsed.dump_lines())) + + def has_confirmed_attendee(self, email: str) -> bool: + """Returns True if the given attendee has accepted an invite to this event""" + parsed = _Component.parse(self.raw) + stack = [parsed] + while stack: + component = stack.pop() + for attendee_line in component.get_all("ATTENDEE"): + sections = attendee_line.split(";") + if f"CN={email}" in sections and "PARTSTAT=ACCEPTED" in sections: + return True + stack.extend(component.subcomponents) + + return False def without_details(self): """Returns a minimal version of this item. @@ -78,8 +92,7 @@ def without_details(self): if subcomp.name != "VTIMEZONE" ] for field in ["DESCRIPTION", "ORGANIZER", "ATTENDEE", "LOCATION"]: - # Repeatedly delete because some fields can appear multiple times - while field in component: + if field in component: del component[field] stack.extend(component.subcomponents) @@ -264,6 +277,17 @@ def _get_item_type(components, wrappers): raise ValueError("Not sure how to join components.") +def _extract_prop_value(line, key): + if line.startswith(key): + prefix_without_params = f"{key}:" + prefix_with_params = f"{key};" + if line.startswith(prefix_without_params): + return line[len(prefix_without_params) :] + elif line.startswith(prefix_with_params): + return line[len(prefix_with_params) :].split(":", 1)[-1] + + return None + class _Component: """ Raw outline of the components. @@ -347,20 +371,15 @@ def dump_lines(self): def __delitem__(self, key): prefix = (f"{key}:", f"{key};") new_lines = [] - lineiter = iter(self.props) - while True: - for line in lineiter: + in_prop = False + for line in iter(self.props): + if not in_prop: if line.startswith(prefix): - break + in_prop = True else: new_lines.append(line) - else: - break - - for line in lineiter: - if not line.startswith((" ", "\t")): - new_lines.append(line) - break + elif not line.startswith((" ", "\t")): + in_prop = False self.props = new_lines @@ -382,26 +401,25 @@ def __contains__(self, obj): raise ValueError(obj) def __getitem__(self, key): - prefix_without_params = f"{key}:" - prefix_with_params = f"{key};" - iterlines = iter(self.props) - for line in iterlines: - if line.startswith(prefix_without_params): - rv = line[len(prefix_without_params) :] - break - elif line.startswith(prefix_with_params): - rv = line[len(prefix_with_params) :].split(":", 1)[-1] - break - else: + try: + return next(self.get_all(key)) + except StopIteration: raise KeyError - - for line in iterlines: - if line.startswith((" ", "\t")): - rv += line[1:] + + def get_all(self, key: str): + rv = None + for line in iter(self.props): + if rv is None: + rv = _extract_prop_value(line, key) else: - break - - return rv + if line.startswith((" ", "\t")): + rv += line[1:] + else: + yield rv + rv = _extract_prop_value(line, key) + + if rv is not None: + yield rv def get(self, key, default=None): try: From 405b37b533a7b653fd55a62ba1cab8d7080cb1a9 Mon Sep 17 00:00:00 2001 From: David Simon Date: Sun, 3 Mar 2024 10:44:41 -0500 Subject: [PATCH 3/4] Fix broken del algorithm --- vdirsyncer/vobject.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/vdirsyncer/vobject.py b/vdirsyncer/vobject.py index aadef784..3718b71e 100644 --- a/vdirsyncer/vobject.py +++ b/vdirsyncer/vobject.py @@ -136,7 +136,8 @@ def ident(self): # with a picture, which bloats the status file. # # 2. The status file would contain really sensitive information. - return self.uid or self.hash + my_ident = self.uid or self.hash + return my_ident @property def parsed(self): @@ -371,15 +372,27 @@ def dump_lines(self): def __delitem__(self, key): prefix = (f"{key}:", f"{key};") new_lines = [] - in_prop = False + + in_target_prop = False for line in iter(self.props): - if not in_prop: + if in_target_prop: + if line.startswith((" ", "\t")): + # Continuing with the prop contents, drop this line + pass + elif line.startswith(prefix): + # Another instance of the target prop, drop this line + pass + else: + # No longer in the target prop, keep this line + in_target_prop = False + new_lines.append(line) + else: if line.startswith(prefix): - in_prop = True + # Entering the target prop, drop this line + in_target_prop = True else: + # Un-targetted prop, keep this line new_lines.append(line) - elif not line.startswith((" ", "\t")): - in_prop = False self.props = new_lines From f65d6a834952e314216251d8edafb1d28b23dc97 Mon Sep 17 00:00:00 2001 From: David Simon Date: Sun, 3 Mar 2024 17:46:16 -0500 Subject: [PATCH 4/4] Properly handle attendee checking on multiple-event items --- vdirsyncer/sync/__init__.py | 6 ++++-- vdirsyncer/vobject.py | 34 +++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/vdirsyncer/sync/__init__.py b/vdirsyncer/sync/__init__.py index fa9df3d2..26d5a14f 100644 --- a/vdirsyncer/sync/__init__.py +++ b/vdirsyncer/sync/__init__.py @@ -67,8 +67,10 @@ def _store_props(ident: str, props: ItemMetadata) -> None: # Prefetch items if prefetch: async for href, item, etag in self.storage.get_multi(prefetch): - if required_attendee and not item.has_confirmed_attendee(required_attendee): - continue + if required_attendee: + item = item.only_with_attendee(required_attendee) + if item is None: + continue if remove_details: item = item.without_details() _store_props( diff --git a/vdirsyncer/vobject.py b/vdirsyncer/vobject.py index 3718b71e..af692f3c 100644 --- a/vdirsyncer/vobject.py +++ b/vdirsyncer/vobject.py @@ -35,6 +35,13 @@ ) +def _includes_attendee(component, attendee_email): + for attendee_line in component.get_all("ATTENDEE"): + sections = attendee_line.split(";") + if f"CN={attendee_email}" in sections and "PARTSTAT=ACCEPTED" in sections: + return True + + class Item: """Immutable wrapper class for VCALENDAR (VEVENT, VTODO) and VCARD""" @@ -56,20 +63,25 @@ def with_uid(self, new_uid): component["UID"] = new_uid return Item("\r\n".join(parsed.dump_lines())) - - def has_confirmed_attendee(self, email: str) -> bool: + + def only_with_attendee(self, email: str): """Returns True if the given attendee has accepted an invite to this event""" parsed = _Component.parse(self.raw) - stack = [parsed] - while stack: - component = stack.pop() - for attendee_line in component.get_all("ATTENDEE"): - sections = attendee_line.split(";") - if f"CN={email}" in sections and "PARTSTAT=ACCEPTED" in sections: - return True - stack.extend(component.subcomponents) - return False + parsed.subcomponents = [ + subcomponent + for subcomponent in parsed.subcomponents + if subcomponent.name != "VEVENT" or _includes_attendee(subcomponent, email) + ] + + if not any( + True + for subcomponent in parsed.subcomponents + if subcomponent.name == "VEVENT" + ): + return None + + return Item("\r\n".join(parsed.dump_lines())) def without_details(self): """Returns a minimal version of this item.