diff --git a/backend/onyx/background/celery/tasks/indexing/tasks.py b/backend/onyx/background/celery/tasks/indexing/tasks.py index 8443bb1f079..9fd73972d0e 100644 --- a/backend/onyx/background/celery/tasks/indexing/tasks.py +++ b/backend/onyx/background/celery/tasks/indexing/tasks.py @@ -278,22 +278,58 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: db_session ) for search_settings_instance in search_settings_list: + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair search settings lock: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + redis_connector_index = redis_connector.new_index( search_settings_instance.id ) if redis_connector_index.fenced: continue + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing get_connector_credential_pair_from_id: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + cc_pair = get_connector_credential_pair_from_id( cc_pair_id, db_session ) if not cc_pair: continue + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing get_last_attempt_for_cc_pair: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + last_attempt = get_last_attempt_for_cc_pair( cc_pair.id, search_settings_instance.id, db_session ) + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair should index: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + search_settings_primary = False if search_settings_instance.id == search_settings_list[0].id: search_settings_primary = True @@ -326,6 +362,15 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: cc_pair.id, None, db_session ) + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair try_creating_indexing_task: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + # using a task queue and only allowing one task per cc_pair/search_setting # prevents us from starving out certain attempts attempt_id = try_creating_indexing_task( @@ -346,6 +391,15 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: ) tasks_created += 1 + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair try_creating_indexing_task finished: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + # debugging logic - remove after we're done if tenant_id in debug_tenants: ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)