diff --git a/packages/block-editor/src/components/button-block-appender/content.scss b/packages/block-editor/src/components/button-block-appender/content.scss
index 941ccb7dd1ad3..14f0f67b11296 100644
--- a/packages/block-editor/src/components/button-block-appender/content.scss
+++ b/packages/block-editor/src/components/button-block-appender/content.scss
@@ -56,7 +56,11 @@
 		}
 
 		.block-editor-inserter {
-			visibility: hidden;
+			opacity: 0;
+
+			&:focus-within {
+				opacity: 1;
+			}
 		}
 
 		&.is-drag-over {
diff --git a/test/e2e/specs/editor/blocks/columns.spec.js b/test/e2e/specs/editor/blocks/columns.spec.js
index bcb23c9a24099..b2c4f51f8fd6e 100644
--- a/test/e2e/specs/editor/blocks/columns.spec.js
+++ b/test/e2e/specs/editor/blocks/columns.spec.js
@@ -311,4 +311,33 @@ test.describe( 'Columns', () => {
 			] );
 		} );
 	} );
+
+	test( 'should arrow up into empty columns', async ( { editor, page } ) => {
+		await editor.insertBlock( {
+			name: 'core/columns',
+			innerBlocks: [ { name: 'core/column' }, { name: 'core/column' } ],
+		} );
+		await editor.insertBlock( {
+			name: 'core/paragraph',
+		} );
+
+		await page.keyboard.press( 'ArrowUp' );
+		await page.keyboard.press( 'ArrowUp' );
+		await page.keyboard.press( 'Delete' );
+
+		await expect.poll( editor.getBlocks ).toMatchObject( [
+			{
+				name: 'core/columns',
+				innerBlocks: [
+					{
+						name: 'core/column',
+					},
+				],
+			},
+			{
+				name: 'core/paragraph',
+				attributes: { content: '' },
+			},
+		] );
+	} );
 } );