GH-39722: [JS] Clean up packaging

js/.vscode/launch.json

@@ -5,24 +5,14 @@
   "version": "0.2.0",
   "inputs": [
     {
-      "type": "pickString",
-      "default": "src",
+      "type": "command",
       "id": "TEST_TARGET",
-      "options": [
-        "src",
-        "apache-arrow",
-        "ts",
-        "es5.cjs",
-        "es5.esm",
-        "es5.umd",
-        "es2015.cjs",
-        "es2015.esm",
-        "es2015.umd",
-        "esnext.cjs",
-        "esnext.esm",
-        "esnext.umd",
-      ],
-      "description": "The JS version + Module format combination to test (or src to test source files)",
+      "command": "shellCommand.execute",
+      "args": {
+        "cwd": "${workspaceFolder}",
+        "description": "The JS version + Module format combination to test (or src to test source files)",
+        "command": "echo \"src\napache-arrow\nts\nes5.cjs\nes5.esm\nes5.umd\nes2015.cjs\nes2015.esm\nes2015.umd\nesnext.cjs\nesnext.esm\nesnext.umd\""
+      }
     },
     {
       "type": "command",
@@ -50,7 +40,7 @@
       "command": "shellCommand.execute",
       "args": {
         "useSingleResult": "true",
-        "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd | apache-arrow) echo '';; *) echo '--experimental-vm-modules';; esac"
+        "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd) echo '--no-warnings';; *) echo '--experimental-vm-modules';; esac"
       }
     },
   ],

js/gulp/arrow-task.js

@@ -30,24 +30,23 @@ import { pipeline } from 'stream/promises';
 
 export const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) {
     const out = targetDir(target);
-    const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`;
+    const dtsGlob = `${targetDir(`es2015`, `esm`)}/**/*.ts`;
     const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`;
     const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
     const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`;
     const esnextUmdGlob = `${targetDir(`esnext`, `umd`)}/*.js`;
-    const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.map`;
+    const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js.map`;
     const esmSourceMapsGlob = `${targetDir(`es2015`, `esm`)}/**/*.map`;
     const es2015UmdSourceMapsGlob = `${targetDir(`es2015`, `umd`)}/*.map`;
     const esnextUmdSourceMapsGlob = `${targetDir(`esnext`, `umd`)}/*.map`;
     return ObservableForkJoin([
         observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files
-        observableFromStreams(gulp.src(dtsGlob), gulpRename((p) => { p.extname = '.mts'; }), gulp.dest(out)), // copy d.ts files as esm
         observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files
         observableFromStreams(gulp.src(cjsSourceMapsGlob), gulp.dest(out)), // copy es2015 cjs sourcemaps
-        observableFromStreams(gulp.src(esmSourceMapsGlob), gulp.dest(out)), // copy es2015 esm sourcemaps
+        observableFromStreams(gulp.src(esmSourceMapsGlob), gulpRename((p) => { p.basename = p.basename.replace('.js', '.mjs'); }), gulpReplace(`.js"`, `.mjs"`), gulp.dest(out)), // copy es2015 esm sourcemaps
         observableFromStreams(gulp.src(es2015UmdSourceMapsGlob), gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename
         observableFromStreams(gulp.src(esnextUmdSourceMapsGlob), gulp.dest(out)), // copy esnext umd sourcemap files, but don't rename
-        observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
+        observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulpReplace(`.js.map`, `.mjs.map`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
         observableFromStreams(gulp.src(es2015UmdGlob), gulpRename((p) => { p.basename += `.es2015.min`; }), gulp.dest(out)), // copy es2015 umd files and add `.es2015.min`
         observableFromStreams(gulp.src(esnextUmdGlob), gulpRename((p) => { p.basename += `.esnext.min`; }), gulp.dest(out)), // copy esnext umd files and add `.esnext.min`
     ]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false }));

js/gulp/closure-task.js

@@ -19,26 +19,34 @@ import { targetDir, mainExport, esmRequire, gCCLanguageNames, publicModulePaths,
 
 import fs from 'node:fs';
 import gulp from 'gulp';
-import path from 'node:path';
+import Path from 'node:path';
+import https from 'node:https';
 import { mkdirp } from 'mkdirp';
+import { PassThrough } from 'node:stream';
 import sourcemaps from 'gulp-sourcemaps';
 import { memoizeTask } from './memoize-task.js';
 import { compileBinFiles } from './typescript-task.js';
 
 import closureCompiler from 'google-closure-compiler';
 const compiler = closureCompiler.gulp();
 
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const closureCompilerVer = JSON.parse(fs.readFileSync(Path.join(__dirname, '..', 'package.json'))).devDependencies['google-closure-compiler'].split('.')[0];
+
 export const closureTask = ((cache) => memoizeTask(cache, async function closure(target, format) {
 
     if (shouldRunInChildProcess(target, format)) {
         return spawnGulpCommandInChildProcess('compile', target, format);
     }
 
     const src = targetDir(target, `cls`);
-    const srcAbsolute = path.resolve(src);
+    const srcAbsolute = Path.resolve(src);
     const out = targetDir(target, format);
-    const externs = path.join(`${out}/${mainExport}.externs.js`);
-    const entry_point = path.join(`${src}/${mainExport}.dom.cls.js`);
+    const externs = Path.join(`${out}/${mainExport}.externs.js`);
+    const entry_point = Path.join(`${src}/${mainExport}.dom.cls.js`);
 
     const exportedImports = publicModulePaths(srcAbsolute).reduce((entries, publicModulePath) => [
         ...entries, {
@@ -54,12 +62,42 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure
         fs.promises.writeFile(entry_point, generateUMDExportAssignment(srcAbsolute, exportedImports))
     ]);
 
-    return await Promise.all([
+    const closeCompilerPolyfills = [];
+
+    await Promise.all([
         runClosureCompileAsObservable().toPromise(),
-        compileBinFiles(target, format).toPromise(),
-        observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)), // copy .d.ts files
+        compileBinFiles(target, format).toPromise().then(() => Promise.all([
+            observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)).toPromise(), // copy .d.ts files,
+            observableFromStreams(gulp.src(`${src}/**/*.d.ts.map`), gulp.dest(out)).toPromise(), // copy .d.ts.map files,
+            observableFromStreams(gulp.src(`${src}/src/**/*`), gulp.dest(`${out}/src`)).toPromise(), // copy TS source files,
+        ]))
     ]);
 
+    // Download the closure compiler polyfill sources for sourcemaps
+    await Promise.all(closeCompilerPolyfills.map(async (path) => {
+
+        await fs.promises.mkdir(
+            Path.join(out, Path.parse(path).dir),
+            { recursive: true, mode: 0o755 }
+        );
+
+        const res = new PassThrough();
+        const req = https.request(
+            new URL(`https://raw.githubusercontent.com/google/closure-compiler/v${closureCompilerVer}/${path}`),
+            (res_) => {
+                if (res_.statusCode === 200) {
+                    res_.pipe(res);
+                } else {
+                    res.end();
+                }
+            }
+        );
+
+        req.on('error', (e) => res.emit('error', e)).end();
+
+        return observableFromStreams(res, fs.createWriteStream(Path.join(out, path))).toPromise();
+    }));
+
     function runClosureCompileAsObservable() {
         return observableFromStreams(
             gulp.src([
@@ -72,8 +110,22 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure
             compiler(createClosureArgs(entry_point, externs, target), {
                 platform: ['native', 'java', 'javascript']
             }),
+            sourcemaps.mapSources((path) => {
+                if (path.indexOf(`${src}/`) === 0) {
+                    return path.slice(`${src}/`.length);
+                }
+                if (path.includes('com/google')) {
+                    closeCompilerPolyfills.push(path);
+                    return path.slice(`src/`.length);
+                }
+                return path;
+            }),
             // rename the sourcemaps from *.js.map files to *.min.js.map
-            sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
+            sourcemaps.write(`./`, {
+                sourceRoot: './src',
+                includeContent: false,
+                mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`),
+            }),
             gulp.dest(out)
         );
     }

js/gulp/package-task.js

@@ -56,7 +56,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
         '.': {
             node: {
                 import: {
-                    types: `./${mainExport}.node.d.mts`,
+                    types: `./${mainExport}.node.d.ts`,
                     default: `./${mainExport}.node.mjs`,
                 },
                 require: {
@@ -65,7 +65,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
                 },
             },
             import: {
-                types: `./${mainExport}.dom.d.mts`,
+                types: `./${mainExport}.dom.d.ts`,
                 default: `./${mainExport}.dom.mjs`,
             },
             require: {
@@ -75,7 +75,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
         },
         './*': {
             import: {
-                types: `./*.d.mts`,
+                types: `./*.d.ts`,
                 default: `./*.mjs`,
             },
             require: {
@@ -127,8 +127,8 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
             sideEffects: format === 'esm' ? false : undefined,
             // include "esm" settings for https://www.npmjs.com/package/esm if building scoped ESM target
             esm:      format === `esm` ? { mode: `auto`, sourceMap: true } : undefined,
-            // set "types" (for TypeScript/VSCode)
-            types:    format === 'umd' ? undefined : `${mainExport}.node.d.ts`,
+            // set "types" to "Arrow.dom" if building scoped UMD target, otherwise "Arrow.node"
+            types:    format === 'umd' ? `${mainExport}.dom.d.ts`: `${mainExport}.node.d.ts`,
         }
     )
 );

js/gulp/typescript-task.js

@@ -61,10 +61,9 @@ function compileTypescript(out, tsconfigPath, tsconfigOverrides, writeSourcemaps
     );
     const writeSources = observableFromStreams(tsProject.src(), gulp.dest(path.join(out, 'src')));
     const writeDTypes = observableFromStreams(dts, sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }), gulp.dest(out));
-    const mapFile = tsProject.options.module === tsc.ModuleKind.ES2015 ? esmMapFile : cjsMapFile;
     const writeJSArgs = writeSourcemaps ? [
         js,
-        sourcemaps.write('./', { mapFile, includeContent: false, sourceRoot: './src' }),
+        sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }),
         gulp.dest(out)
       ] : [
         js,
@@ -73,6 +72,3 @@ function compileTypescript(out, tsconfigPath, tsconfigOverrides, writeSourcemaps
     const writeJS = observableFromStreams(...writeJSArgs);
     return ObservableForkJoin([writeSources, writeDTypes, writeJS]);
 }
-
-const cjsMapFile = (mapFilePath) => mapFilePath;
-const esmMapFile = (mapFilePath) => mapFilePath.replace('.js.map', '.mjs.map');