diff --git a/.changeset/dull-pants-return.md b/.changeset/dull-pants-return.md
new file mode 100644
index 000000000000..31c13cd95a1d
--- /dev/null
+++ b/.changeset/dull-pants-return.md
@@ -0,0 +1,5 @@
+---
+"@sveltejs/kit": patch
+---
+
+Build server without removing `sveltekit` Vite plugin
diff --git a/packages/kit/src/vite/build/build_server.js b/packages/kit/src/vite/build/build_server.js
index 8772002705ab..03cfba500ffd 100644
--- a/packages/kit/src/vite/build/build_server.js
+++ b/packages/kit/src/vite/build/build_server.js
@@ -4,13 +4,7 @@ import { mkdirp, posixify } from '../../utils/filesystem.js';
 import { get_vite_config, merge_vite_configs, resolve_entry } from '../utils.js';
 import { load_template } from '../../core/config/index.js';
 import { get_runtime_directory } from '../../core/utils.js';
-import {
-	create_build,
-	find_deps,
-	get_default_config,
-	remove_svelte_kit,
-	is_http_method
-} from './utils.js';
+import { create_build, find_deps, get_default_config, is_http_method } from './utils.js';
 import { s } from '../../utils/misc.js';
 
 /**
@@ -215,8 +209,6 @@ export async function build_server(options, client) {
 		await get_vite_config(vite_config, vite_config_env)
 	);
 
-	remove_svelte_kit(merged_config);
-
 	const { chunks } = await create_build(merged_config);
 
 	/** @type {import('vite').Manifest} */
diff --git a/packages/kit/src/vite/build/build_service_worker.js b/packages/kit/src/vite/build/build_service_worker.js
index 19a7134829c2..acc1e0a76bde 100644
--- a/packages/kit/src/vite/build/build_service_worker.js
+++ b/packages/kit/src/vite/build/build_service_worker.js
@@ -1,8 +1,7 @@
 import fs from 'fs';
 import * as vite from 'vite';
 import { s } from '../../utils/misc.js';
-import { get_vite_config, merge_vite_configs } from '../utils.js';
-import { assets_base, remove_svelte_kit } from './utils.js';
+import { assets_base } from './utils.js';
 
 /**
  * @param {{
@@ -17,7 +16,7 @@ import { assets_base, remove_svelte_kit } from './utils.js';
  * @param {import('vite').Manifest} client_manifest
  */
 export async function build_service_worker(
-	{ config, vite_config, vite_config_env, manifest_data, output_dir, service_worker_entry_file },
+	{ config, manifest_data, output_dir, service_worker_entry_file },
 	prerendered,
 	client_manifest
 ) {
@@ -64,7 +63,7 @@ export async function build_service_worker(
 			.trim()
 	);
 
-	const merged_config = merge_vite_configs(await get_vite_config(vite_config, vite_config_env), {
+	await vite.build({
 		base: assets_base(config.kit),
 		build: {
 			lib: {
@@ -80,7 +79,6 @@ export async function build_service_worker(
 			outDir: `${output_dir}/client`,
 			emptyOutDir: false
 		},
-		// @ts-expect-error
 		configFile: false,
 		resolve: {
 			alias: {
@@ -89,8 +87,4 @@ export async function build_service_worker(
 			}
 		}
 	});
-
-	remove_svelte_kit(merged_config);
-
-	await vite.build(merged_config);
 }
diff --git a/packages/kit/src/vite/build/utils.js b/packages/kit/src/vite/build/utils.js
index 614a07b79a40..523e09fbbf97 100644
--- a/packages/kit/src/vite/build/utils.js
+++ b/packages/kit/src/vite/build/utils.js
@@ -147,20 +147,6 @@ export function assets_base(config) {
 	return `${assets || base}/`;
 }
 
-/**
- * vite.config.js will contain vite-plugin-svelte-kit, which kicks off the server and service
- * worker builds in a hook. When running the server and service worker builds we must remove
- * the SvelteKit plugin so that we do not kick off additional instances of these builds.
- * @param {import('vite').UserConfig} config
- */
-export function remove_svelte_kit(config) {
-	// TODO i feel like there's a more elegant way to do this
-	// @ts-expect-error - it can't handle infinite type expansion
-	config.plugins = (config.plugins || [])
-		.flat(Infinity)
-		.filter((plugin) => plugin.name !== 'vite-plugin-svelte-kit');
-}
-
 const method_names = new Set(['GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH']);
 
 // If we'd written this in TypeScript, it could be easy...
diff --git a/packages/kit/src/vite/index.js b/packages/kit/src/vite/index.js
index e6d347d66b9b..5fc4f78d3bd0 100644
--- a/packages/kit/src/vite/index.js
+++ b/packages/kit/src/vite/index.js
@@ -180,6 +180,11 @@ function kit() {
 		 * @see https://vitejs.dev/guide/api-plugin.html#config
 		 */
 		async config(config, config_env) {
+			// The config is created in build_server for SSR mode and passed inline
+			if (config.build?.ssr) {
+				return;
+			}
+
 			vite_config_env = config_env;
 			svelte_config = await load_config();
 			is_build = config_env.command === 'build';
@@ -264,6 +269,10 @@ function kit() {
 		 * Clears the output directories.
 		 */
 		buildStart() {
+			if (vite_config.build.ssr) {
+				return;
+			}
+
 			// Reset for new build. Goes here because `build --watch` calls buildStart but not config
 			completed_build = false;
 
@@ -282,6 +291,10 @@ function kit() {
 		 * then use this hook to kick off builds for the server and service worker.
 		 */
 		async writeBundle(_options, bundle) {
+			if (vite_config.build.ssr) {
+				return;
+			}
+
 			for (const file of manifest_data.components) {
 				const id = vite.normalizePath(path.resolve(file));
 				const node = this.getModuleInfo(id);
@@ -398,9 +411,10 @@ function kit() {
 		 * Runs the adapter.
 		 */
 		async closeBundle() {
-			if (!completed_build) {
-				// vite calls closeBundle when dev-server restarts, ignore that,
-				// and only adapt when build successfully completes.
+			// vite calls closeBundle when dev-server restarts, ignore that,
+			// and only adapt when build successfully completes.
+			const is_restart = !completed_build;
+			if (vite_config.build.ssr || is_restart) {
 				return;
 			}