Move encoding to common for #79275 (#100539)

* move encoding.ts to common for #79275 * load iconv-lite-umd and jschardet in web version for #79275 * move EncodingOracle to the AbstractTextFileService for #79275 * review * update to new iconv-lite-umd * add workaround for jschardet types * fix indentation * add iconv-lite-umd and jschardet to workbench.html * fix paths for modules Co-authored-by: Benjamin Pasero <benjpas@microsoft.com>
microsoft · Jun 20, 2020 · 24e0a82 · 24e0a82
1 parent aa06a3d
commit 24e0a82
Show file tree

Hide file tree

Showing 39 changed files with 249 additions and 205 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -519,7 +519,8 @@
                     "**/vs/workbench/services/**/common/**",
                     "**/vs/workbench/api/**/common/**",
                     "vscode-textmate",
-                    "vscode-oniguruma"
+                    "vscode-oniguruma",
+                    "iconv-lite-umd"
                 ]
             },
             {

diff --git a/build/lib/i18n.js b/build/lib/i18n.js
@@ -1175,9 +1175,10 @@ function createIslFile(originalFilePath, messages, language, innoSetup) {
     });
     const basename = path.basename(originalFilePath);
     const filePath = `${basename}.${language.id}.isl`;
+    const encoded = iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage);
     return new File({
         path: filePath,
-        contents: iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage)
+        contents: Buffer.from(encoded),
     });
 }
 function encodeEntities(value) {

diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts
@@ -1339,10 +1339,11 @@ function createIslFile(originalFilePath: string, messages: Map<string>, language
 
 	const basename = path.basename(originalFilePath);
 	const filePath = `${basename}.${language.id}.isl`;
+	const encoded = iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage);
 
 	return new File({
 		path: filePath,
-		contents: iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage)
+		contents: Buffer.from(encoded),
 	});
 }
 

diff --git a/build/package.json b/build/package.json
@@ -38,7 +38,7 @@
     "gulp-bom": "^1.0.0",
     "gulp-sourcemaps": "^1.11.0",
     "gulp-uglify": "^3.0.0",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "mime": "^1.3.4",
     "minimatch": "3.0.4",
     "minimist": "^1.2.3",

diff --git a/build/yarn.lock b/build/yarn.lock
@@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 ignore@^5.1.1:
   version "5.1.2"

diff --git a/extensions/git/package.json b/extensions/git/package.json
@@ -1878,7 +1878,7 @@
   "dependencies": {
     "byline": "^5.0.0",
     "file-type": "^7.2.0",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "jschardet": "2.1.1",
     "vscode-extension-telemetry": "0.1.1",
     "vscode-nls": "^4.0.0",

diff --git a/extensions/git/yarn.lock b/extensions/git/yarn.lock
@@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 inflight@^1.0.4:
   version "1.0.6"

diff --git a/package.json b/package.json
@@ -41,7 +41,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "jschardet": "2.1.1",
     "keytar": "^5.5.0",
     "minimist": "^1.2.5",

diff --git a/remote/package.json b/remote/package.json
@@ -8,7 +8,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "jschardet": "2.1.1",
     "minimist": "^1.2.5",
     "native-watchdog": "1.3.0",

diff --git a/remote/web/package.json b/remote/web/package.json
@@ -3,6 +3,8 @@
 	"version": "0.0.0",
 	"dependencies": {
 		"semver-umd": "^5.5.7",
+		"iconv-lite-umd": "0.6.3",
+		"jschardet": "2.1.1",
 		"vscode-oniguruma": "1.3.1",
 		"vscode-textmate": "5.1.1",
 		"xterm": "4.7.0-beta.3",

diff --git a/remote/web/yarn.lock b/remote/web/yarn.lock
@@ -2,6 +2,16 @@
 # yarn lockfile v1
 
 
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
+
+jschardet@2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/jschardet/-/jschardet-2.1.1.tgz#af6f8fd0b3b0f5d46a8fd9614a4fce490575c184"
+  integrity sha512-pA5qG9Zwm8CBpGlK/lo2GE9jPxwqRgMV7Lzc/1iaPccw6v4Rhj8Zg2BTyrdmHmxlJojnbLupLeRnaPLsq03x6Q==
+
 semver-umd@^5.5.7:
   version "5.5.7"
   resolved "https://registry.yarnpkg.com/semver-umd/-/semver-umd-5.5.7.tgz#966beb5e96c7da6fbf09c3da14c2872d6836c528"

diff --git a/remote/yarn.lock b/remote/yarn.lock
@@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 ip@^1.1.5:
   version "1.1.5"

diff --git a/src/vs/code/browser/workbench/workbench-dev.html b/src/vs/code/browser/workbench/workbench-dev.html
@@ -37,6 +37,8 @@
 				'xterm-addon-unicode11': `${window.location.origin}/static/remote/web/node_modules/xterm-addon-unicode11/lib/xterm-addon-unicode11.js`,
 				'xterm-addon-webgl': `${window.location.origin}/static/remote/web/node_modules/xterm-addon-webgl/lib/xterm-addon-webgl.js`,
 				'semver-umd': `${window.location.origin}/static/remote/web/node_modules/semver-umd/lib/semver-umd.js`,
+				'iconv-lite-umd': `${window.location.origin}/static/remote/web/node_modules/iconv-lite-umd/lib/iconv-lite-umd.js`,
+				'jschardet': `${window.location.origin}/static/remote/web/node_modules/jschardet/dist/jschardet.min.js`,
 			}
 		};
 	</script>

diff --git a/src/vs/code/browser/workbench/workbench.html b/src/vs/code/browser/workbench/workbench.html
@@ -23,6 +23,8 @@
 
 		<!-- Prefetch to avoid waterfall -->
 		<link rel="prefetch" href="./static/node_modules/semver-umd/lib/semver-umd.js">
+		<link rel="prefetch" href="./static/node_modules/iconv-lite-umd/lib/iconv-lite-umd.js">
+		<link rel="prefetch" href="./static/node_modules/jschardet/dist/jschardet.min.js">
 	</head>
 
 	<body aria-label="">
@@ -41,6 +43,8 @@
 				'xterm-addon-unicode11': `${window.location.origin}/static/node_modules/xterm-addon-unicode11/lib/xterm-addon-unicode11.js`,
 				'xterm-addon-webgl': `${window.location.origin}/static/node_modules/xterm-addon-webgl/lib/xterm-addon-webgl.js`,
 				'semver-umd': `${window.location.origin}/static/node_modules/semver-umd/lib/semver-umd.js`,
+				'iconv-lite-umd': `${window.location.origin}/static/node_modules/iconv-lite-umd/lib/iconv-lite-umd.js`,
+				'jschardet': `${window.location.origin}/static/node_modules/jschardet/dist/jschardet.min.js`,
 			}
 		};
 	</script>

diff --git a/src/vs/workbench/services/search/node/textSearchManager.ts b/src/vs/workbench/services/search/node/textSearchManager.ts
@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { toCanonicalName } from 'vs/base/node/encoding';
+import { toCanonicalName } from 'vs/workbench/services/textfile/common/encoding';
 import * as pfs from 'vs/base/node/pfs';
 import { ITextQuery } from 'vs/workbench/services/search/common/search';
 import { TextSearchProvider } from 'vs/workbench/services/search/common/searchExtTypes';

diff --git a/src/vs/workbench/services/textfile/browser/browserTextFileService.ts b/src/vs/workbench/services/textfile/browser/browserTextFileService.ts
@@ -3,18 +3,22 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { AbstractTextFileService } from 'vs/workbench/services/textfile/browser/textFileService';
-import { ITextFileService, IResourceEncodings, IResourceEncoding, TextFileEditorModelState } from 'vs/workbench/services/textfile/common/textfiles';
+import { AbstractTextFileService, EncodingOracle } from 'vs/workbench/services/textfile/browser/textFileService';
+import { ITextFileService, IResourceEncoding, TextFileEditorModelState } from 'vs/workbench/services/textfile/common/textfiles';
 import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
 import { ShutdownReason } from 'vs/platform/lifecycle/common/lifecycle';
 
 export class BrowserTextFileService extends AbstractTextFileService {
 
-	readonly encoding: IResourceEncodings = {
-		async getPreferredWriteEncoding(): Promise<IResourceEncoding> {
-			return { encoding: 'utf8', hasBOM: false };
+	private _browserEncoding: EncodingOracle | undefined;
+
+	get encoding(): EncodingOracle {
+		if (!this._browserEncoding) {
+			this._browserEncoding = this._register(this.instantiationService.createInstance(BrowserEncodingOracle));
 		}
-	};
+
+		return this._browserEncoding;
+	}
 
 	protected registerListeners(): void {
 		super.registerListeners();
@@ -34,4 +38,18 @@ export class BrowserTextFileService extends AbstractTextFileService {
 	}
 }
 
+class BrowserEncodingOracle extends EncodingOracle {
+	async getPreferredWriteEncoding(): Promise<IResourceEncoding> {
+		return { encoding: 'utf8', hasBOM: false };
+	}
+
+	async getWriteEncoding(): Promise<{ encoding: string, addBOM: boolean }> {
+		return { encoding: 'utf8', addBOM: false };
+	}
+
+	async getReadEncoding(): Promise<string> {
+		return 'utf8';
+	}
+}
+
 registerSingleton(ITextFileService, BrowserTextFileService);