Skip to content

Commit

Permalink
Mitigate permissions loss in Table ACLs by running appliers with sing…
Browse files Browse the repository at this point in the history
…le thread (#518)
  • Loading branch information
mwojtyczka authored Oct 27, 2023
1 parent 226f078 commit 18e3b54
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 5 deletions.
11 changes: 7 additions & 4 deletions src/databricks/labs/ucx/framework/parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,13 @@ def __init__(self, name, tasks: list[Callable[..., Result]], num_threads: int):
self._default_log_every = 100

@classmethod
def gather(cls, name: str, tasks: list[Callable[..., Result]]) -> (list[Result], list[Exception]):
num_threads = os.cpu_count() * 2
if num_threads < MIN_THREADS:
num_threads = MIN_THREADS
def gather(
cls, name: str, tasks: list[Callable[..., Result]], num_threads: int | None = None
) -> (list[Result], list[Exception]):
if num_threads is None:
num_threads = os.cpu_count() * 2
if num_threads < MIN_THREADS:
num_threads = MIN_THREADS
return cls(name, tasks, num_threads=num_threads)._run()

def _run(self) -> (list[Result], list[Exception]):
Expand Down
4 changes: 3 additions & 1 deletion src/databricks/labs/ucx/workspace_access/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ def apply_group_permissions(self, migration_state: GroupMigrationState, destinat
applier_tasks.extend(tasks_for_support)

logger.info(f"Starting to apply permissions on {destination} groups. Total tasks: {len(applier_tasks)}")
_, errors = Threads.gather(f"apply {destination} group permissions", applier_tasks)
# run with 1 thread to temp bugfix TableACL grants issue in the platform
# see: https://databricks.atlassian.net/browse/ES-908737
_, errors = Threads.gather(f"apply {destination} group permissions", applier_tasks, num_threads=1)
if len(errors) > 0:
# TODO: https://github.com/databrickslabs/ucx/issues/406
logger.error(f"Detected {len(errors)} while applying permissions")
Expand Down

0 comments on commit 18e3b54

Please sign in to comment.