diff --git a/packages/block-editor/src/components/block-toolbar/index.js b/packages/block-editor/src/components/block-toolbar/index.js
index 94de64fc91492..0f968b8a209a7 100644
--- a/packages/block-editor/src/components/block-toolbar/index.js
+++ b/packages/block-editor/src/components/block-toolbar/index.js
@@ -129,9 +129,9 @@ const BlockToolbar = ( { hideDragHandle } ) => {
 			{ ! isMultiToolbar &&
 				isLargeViewport &&
 				blockEditingMode === 'default' && <BlockParentSelector /> }
-			<div ref={ nodeRef } { ...showMoversGestures }>
-				{ ( shouldShowVisualToolbar || isMultiToolbar ) &&
-					blockEditingMode === 'default' && (
+			{ ( shouldShowVisualToolbar || isMultiToolbar ) &&
+				blockEditingMode === 'default' && (
+					<div ref={ nodeRef } { ...showMoversGestures }>
 						<ToolbarGroup className="block-editor-block-toolbar__block-controls">
 							<BlockSwitcher clientIds={ blockClientIds } />
 							{ ! isMultiToolbar && (
@@ -145,8 +145,8 @@ const BlockToolbar = ( { hideDragHandle } ) => {
 								hideDragHandle={ hideDragHandle }
 							/>
 						</ToolbarGroup>
-					) }
-			</div>
+					</div>
+				) }
 			{ shouldShowVisualToolbar && isMultiToolbar && (
 				<BlockGroupToolbar />
 			) }
diff --git a/packages/block-editor/src/components/block-tools/style.scss b/packages/block-editor/src/components/block-tools/style.scss
index cf38c9ee47de2..c55b8e651c2e7 100644
--- a/packages/block-editor/src/components/block-tools/style.scss
+++ b/packages/block-editor/src/components/block-tools/style.scss
@@ -121,6 +121,10 @@
 		}
 	}
 
+	&:has(.block-editor-block-toolbar:empty) {
+		display: none;
+	}
+
 	// on desktop and tablet viewports the toolbar is fixed
 	// on top of interface header