diff --git a/packages/block-library/src/image/edit.js b/packages/block-library/src/image/edit.js
index b7dba1d0f2db98..dd938ee6860d66 100644
--- a/packages/block-library/src/image/edit.js
+++ b/packages/block-library/src/image/edit.js
@@ -161,6 +161,7 @@ export function ImageEdit( {
 				alt: undefined,
 				id: undefined,
 				title: undefined,
+				caption: undefined,
 			} );
 			return;
 		}
@@ -175,9 +176,9 @@ export function ImageEdit( {
 			}
 		}
 
-		// If a caption text was meanwhile written by the user, make sure the
-		// text is not overwritten by another caption.
-		if ( caption ) {
+		// If a caption text was meanwhile written by the user,
+		// make sure the text is not overwritten by empty captions.
+		if ( caption && ! get( mediaAttributes, [ 'caption' ] ) ) {
 			mediaAttributes = omit( mediaAttributes, [ 'caption' ] );
 		}
 
@@ -403,33 +404,11 @@ export function ImageEdit( {
 		imageHeight,
 	} = useImageSize( ref, url, [ align ] );
 
-	let captionField;
-
-	if ( ! RichText.isEmpty( caption ) || isSelected ) {
-		captionField = (
-			<RichText
-				tagName="figcaption"
-				placeholder={ __( 'Write caption…' ) }
-				value={ caption }
-				unstableOnFocus={ onFocusCaption }
-				onChange={ ( value ) => setAttributes( { caption: value } ) }
-				isSelected={ captionFocused }
-				inlineToolbar
-				__unstableOnSplitAtEnd={ () =>
-					insertBlocksAfter( createBlock( 'core/paragraph' ) )
-				}
-			/>
-		);
-	}
-
 	if ( ! url ) {
 		return (
 			<>
 				{ controls }
-				<Block.figure>
-					{ mediaPlaceholder }
-					{ captionField }
-				</Block.figure>
+				<Block.div>{ mediaPlaceholder }</Block.div>
 			</>
 		);
 	}
@@ -617,7 +596,22 @@ export function ImageEdit( {
 			{ controls }
 			<Block.figure ref={ ref } className={ classes }>
 				{ img }
-				{ captionField }
+				{ ( ! RichText.isEmpty( caption ) || isSelected ) && (
+					<RichText
+						tagName="figcaption"
+						placeholder={ __( 'Write caption…' ) }
+						value={ caption }
+						unstableOnFocus={ onFocusCaption }
+						onChange={ ( value ) =>
+							setAttributes( { caption: value } )
+						}
+						isSelected={ captionFocused }
+						inlineToolbar
+						__unstableOnSplitAtEnd={ () =>
+							insertBlocksAfter( createBlock( 'core/paragraph' ) )
+						}
+					/>
+				) }
 				{ mediaPlaceholder }
 			</Block.figure>
 		</>
diff --git a/packages/e2e-tests/specs/editor/blocks/image.test.js b/packages/e2e-tests/specs/editor/blocks/image.test.js
index bb9127494e306e..3e93c7a340fdaa 100644
--- a/packages/e2e-tests/specs/editor/blocks/image.test.js
+++ b/packages/e2e-tests/specs/editor/blocks/image.test.js
@@ -98,15 +98,4 @@ describe( 'Image', () => {
 			await page.evaluate( () => document.activeElement.innerHTML )
 		).toBe( '12' );
 	} );
-
-	it( 'should keep caption after upload', async () => {
-		await insertBlock( 'Image' );
-		await page.keyboard.type( '1' );
-		const filename1 = await upload( '.wp-block-image input[type="file"]' );
-
-		const regex = new RegExp(
-			`<!-- wp:image {"id":\\d+,"sizeSlug":"large"} -->\\s*<figure class="wp-block-image size-large"><img src="[^"]+\\/${ filename1 }\\.png" alt="" class="wp-image-\\d+"/><figcaption>1</figcaption></figure>\\s*<!-- \\/wp:image -->`
-		);
-		expect( await getEditedPostContent() ).toMatch( regex );
-	} );
 } );
diff --git a/packages/e2e-tests/specs/editor/plugins/templates.test.js b/packages/e2e-tests/specs/editor/plugins/templates.test.js
index 5983fb0390a313..a738ffae94da2a 100644
--- a/packages/e2e-tests/specs/editor/plugins/templates.test.js
+++ b/packages/e2e-tests/specs/editor/plugins/templates.test.js
@@ -37,7 +37,6 @@ describe( 'templates', () => {
 			// re-added after saving and reloading the editor.
 			await page.click( '.editor-post-title__input' );
 			await page.keyboard.press( 'ArrowDown' );
-			await pressKeyWithModifier( 'shift', 'Tab' );
 			await page.keyboard.press( 'Backspace' );
 			await saveDraft();
 			await page.reload();
@@ -50,7 +49,6 @@ describe( 'templates', () => {
 			// re-added after saving and reloading the editor.
 			await page.type( '.editor-post-title__input', 'My Empty Book' );
 			await page.keyboard.press( 'ArrowDown' );
-			await pressKeyWithModifier( 'shift', 'Tab' );
 			await pressKeyWithModifier( 'primary', 'A' );
 			await page.keyboard.press( 'Backspace' );
 			await saveDraft();
@@ -102,7 +100,6 @@ describe( 'templates', () => {
 			await page.type( '.editor-post-title__input', 'My Image Format' );
 			await clickBlockAppender();
 			await page.keyboard.press( 'Backspace' );
-			await pressKeyWithModifier( 'shift', 'Tab' );
 			await page.keyboard.press( 'Backspace' );
 			await saveDraft();
 			await page.reload();
diff --git a/packages/e2e-tests/specs/editor/various/adding-blocks.test.js b/packages/e2e-tests/specs/editor/various/adding-blocks.test.js
index 50489e406442f0..6e222ff24e5b79 100644
--- a/packages/e2e-tests/specs/editor/various/adding-blocks.test.js
+++ b/packages/e2e-tests/specs/editor/various/adding-blocks.test.js
@@ -8,7 +8,6 @@ import {
 	pressKeyTimes,
 	setBrowserViewport,
 	closeGlobalBlockInserter,
-	pressKeyWithModifier,
 } from '@wordpress/e2e-test-utils';
 
 /** @typedef {import('puppeteer').ElementHandle} ElementHandle */
@@ -67,7 +66,6 @@ describe( 'adding blocks', () => {
 		await page.keyboard.press( 'Enter' );
 		expect( await getEditedPostContent() ).toMatchSnapshot();
 		await page.keyboard.press( 'Backspace' );
-		await pressKeyWithModifier( 'shift', 'Tab' );
 		await page.keyboard.press( 'Backspace' );
 		expect( await getEditedPostContent() ).toMatchSnapshot();
 
diff --git a/packages/e2e-tests/specs/editor/various/block-deletion.test.js b/packages/e2e-tests/specs/editor/various/block-deletion.test.js
index 207d06869da6c1..00027d94137cf2 100644
--- a/packages/e2e-tests/specs/editor/various/block-deletion.test.js
+++ b/packages/e2e-tests/specs/editor/various/block-deletion.test.js
@@ -118,9 +118,6 @@ describe( 'block deletion -', () => {
 			await page.click(
 				'.wp-block[data-type="core/image"] .components-placeholder__label'
 			);
-			await page.click(
-				'.wp-block[data-type="core/image"] .components-placeholder__label'
-			);
 			await page.keyboard.press( 'Backspace' );
 
 			expect( await getEditedPostContent() ).toMatchSnapshot();
@@ -199,7 +196,6 @@ describe( 'deleting all blocks', () => {
 
 		// Add and remove a block.
 		await insertBlock( 'Image' );
-		await pressKeyWithModifier( 'shift', 'Tab' );
 		await page.keyboard.press( 'Backspace' );
 
 		// Verify there is no selected block.