From 8348c538a5354127e95bf37157040138254b351c Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Wed, 17 Apr 2024 23:21:42 -0400 Subject: [PATCH] GH-39722: [JS] Clean up packaging (#39723) * Remove .mts since it's the same as .ts * Use .js consistently for all packages * Closes: #39722 --------- Signed-off-by: Matt Topol Co-authored-by: ptaylor Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- js/.vscode/launch.json | 26 +++++---------- js/gulp/arrow-task.js | 9 +++-- js/gulp/closure-task.js | 68 +++++++++++++++++++++++++++++++++----- js/gulp/package-task.js | 10 +++--- js/gulp/typescript-task.js | 6 +--- 5 files changed, 78 insertions(+), 41 deletions(-) diff --git a/js/.vscode/launch.json b/js/.vscode/launch.json index 21255b61c39a5..189b9d1f33c27 100644 --- a/js/.vscode/launch.json +++ b/js/.vscode/launch.json @@ -5,24 +5,14 @@ "version": "0.2.0", "inputs": [ { - "type": "pickString", - "default": "src", + "type": "command", "id": "TEST_TARGET", - "options": [ - "src", - "apache-arrow", - "ts", - "es5.cjs", - "es5.esm", - "es5.umd", - "es2015.cjs", - "es2015.esm", - "es2015.umd", - "esnext.cjs", - "esnext.esm", - "esnext.umd", - ], - "description": "The JS version + Module format combination to test (or src to test source files)", + "command": "shellCommand.execute", + "args": { + "cwd": "${workspaceFolder}", + "description": "The JS version + Module format combination to test (or src to test source files)", + "command": "echo \"src\napache-arrow\nts\nes5.cjs\nes5.esm\nes5.umd\nes2015.cjs\nes2015.esm\nes2015.umd\nesnext.cjs\nesnext.esm\nesnext.umd\"" + } }, { "type": "command", @@ -50,7 +40,7 @@ "command": "shellCommand.execute", "args": { "useSingleResult": "true", - "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd | apache-arrow) echo '';; *) echo '--experimental-vm-modules';; esac" + "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd) echo '--no-warnings';; *) echo '--experimental-vm-modules';; esac" } }, ], diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js index 855cb71283748..2ec2043985074 100644 --- a/js/gulp/arrow-task.js +++ b/js/gulp/arrow-task.js @@ -30,24 +30,23 @@ import { pipeline } from 'stream/promises'; export const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) { const out = targetDir(target); - const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`; + const dtsGlob = `${targetDir(`es2015`, `esm`)}/**/*.ts`; const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`; const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`; const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`; const esnextUmdGlob = `${targetDir(`esnext`, `umd`)}/*.js`; - const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.map`; + const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js.map`; const esmSourceMapsGlob = `${targetDir(`es2015`, `esm`)}/**/*.map`; const es2015UmdSourceMapsGlob = `${targetDir(`es2015`, `umd`)}/*.map`; const esnextUmdSourceMapsGlob = `${targetDir(`esnext`, `umd`)}/*.map`; return ObservableForkJoin([ observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files - observableFromStreams(gulp.src(dtsGlob), gulpRename((p) => { p.extname = '.mts'; }), gulp.dest(out)), // copy d.ts files as esm observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files observableFromStreams(gulp.src(cjsSourceMapsGlob), gulp.dest(out)), // copy es2015 cjs sourcemaps - observableFromStreams(gulp.src(esmSourceMapsGlob), gulp.dest(out)), // copy es2015 esm sourcemaps + observableFromStreams(gulp.src(esmSourceMapsGlob), gulpRename((p) => { p.basename = p.basename.replace('.js', '.mjs'); }), gulpReplace(`.js"`, `.mjs"`), gulp.dest(out)), // copy es2015 esm sourcemaps observableFromStreams(gulp.src(es2015UmdSourceMapsGlob), gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename observableFromStreams(gulp.src(esnextUmdSourceMapsGlob), gulp.dest(out)), // copy esnext umd sourcemap files, but don't rename - observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs` + observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulpReplace(`.js.map`, `.mjs.map`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs` observableFromStreams(gulp.src(es2015UmdGlob), gulpRename((p) => { p.basename += `.es2015.min`; }), gulp.dest(out)), // copy es2015 umd files and add `.es2015.min` observableFromStreams(gulp.src(esnextUmdGlob), gulpRename((p) => { p.basename += `.esnext.min`; }), gulp.dest(out)), // copy esnext umd files and add `.esnext.min` ]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })); diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js index c620784fd1a84..80f841bf729f1 100644 --- a/js/gulp/closure-task.js +++ b/js/gulp/closure-task.js @@ -19,8 +19,10 @@ import { targetDir, mainExport, esmRequire, gCCLanguageNames, publicModulePaths, import fs from 'node:fs'; import gulp from 'gulp'; -import path from 'node:path'; +import Path from 'node:path'; +import https from 'node:https'; import { mkdirp } from 'mkdirp'; +import { PassThrough } from 'node:stream'; import sourcemaps from 'gulp-sourcemaps'; import { memoizeTask } from './memoize-task.js'; import { compileBinFiles } from './typescript-task.js'; @@ -28,6 +30,12 @@ import { compileBinFiles } from './typescript-task.js'; import closureCompiler from 'google-closure-compiler'; const compiler = closureCompiler.gulp(); +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const closureCompilerVer = JSON.parse(fs.readFileSync(Path.join(__dirname, '..', 'package.json'))).devDependencies['google-closure-compiler'].split('.')[0]; + export const closureTask = ((cache) => memoizeTask(cache, async function closure(target, format) { if (shouldRunInChildProcess(target, format)) { @@ -35,10 +43,10 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure } const src = targetDir(target, `cls`); - const srcAbsolute = path.resolve(src); + const srcAbsolute = Path.resolve(src); const out = targetDir(target, format); - const externs = path.join(`${out}/${mainExport}.externs.js`); - const entry_point = path.join(`${src}/${mainExport}.dom.cls.js`); + const externs = Path.join(`${out}/${mainExport}.externs.js`); + const entry_point = Path.join(`${src}/${mainExport}.dom.cls.js`); const exportedImports = publicModulePaths(srcAbsolute).reduce((entries, publicModulePath) => [ ...entries, { @@ -54,12 +62,42 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure fs.promises.writeFile(entry_point, generateUMDExportAssignment(srcAbsolute, exportedImports)) ]); - return await Promise.all([ + const closeCompilerPolyfills = []; + + await Promise.all([ runClosureCompileAsObservable().toPromise(), - compileBinFiles(target, format).toPromise(), - observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)), // copy .d.ts files + compileBinFiles(target, format).toPromise().then(() => Promise.all([ + observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)).toPromise(), // copy .d.ts files, + observableFromStreams(gulp.src(`${src}/**/*.d.ts.map`), gulp.dest(out)).toPromise(), // copy .d.ts.map files, + observableFromStreams(gulp.src(`${src}/src/**/*`), gulp.dest(`${out}/src`)).toPromise(), // copy TS source files, + ])) ]); + // Download the closure compiler polyfill sources for sourcemaps + await Promise.all(closeCompilerPolyfills.map(async (path) => { + + await fs.promises.mkdir( + Path.join(out, Path.parse(path).dir), + { recursive: true, mode: 0o755 } + ); + + const res = new PassThrough(); + const req = https.request( + new URL(`https://raw.githubusercontent.com/google/closure-compiler/v${closureCompilerVer}/${path}`), + (res_) => { + if (res_.statusCode === 200) { + res_.pipe(res); + } else { + res.end(); + } + } + ); + + req.on('error', (e) => res.emit('error', e)).end(); + + return observableFromStreams(res, fs.createWriteStream(Path.join(out, path))).toPromise(); + })); + function runClosureCompileAsObservable() { return observableFromStreams( gulp.src([ @@ -72,8 +110,22 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure compiler(createClosureArgs(entry_point, externs, target), { platform: ['native', 'java', 'javascript'] }), + sourcemaps.mapSources((path) => { + if (path.indexOf(`${src}/`) === 0) { + return path.slice(`${src}/`.length); + } + if (path.includes('com/google')) { + closeCompilerPolyfills.push(path); + return path.slice(`src/`.length); + } + return path; + }), // rename the sourcemaps from *.js.map files to *.min.js.map - sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }), + sourcemaps.write(`./`, { + sourceRoot: './src', + includeContent: false, + mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`), + }), gulp.dest(out) ); } diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js index 9bc002e664885..0b0f4cfa20b8b 100644 --- a/js/gulp/package-task.js +++ b/js/gulp/package-task.js @@ -56,7 +56,7 @@ const createMainPackageJson = (target, format) => (orig) => ({ '.': { node: { import: { - types: `./${mainExport}.node.d.mts`, + types: `./${mainExport}.node.d.ts`, default: `./${mainExport}.node.mjs`, }, require: { @@ -65,7 +65,7 @@ const createMainPackageJson = (target, format) => (orig) => ({ }, }, import: { - types: `./${mainExport}.dom.d.mts`, + types: `./${mainExport}.dom.d.ts`, default: `./${mainExport}.dom.mjs`, }, require: { @@ -75,7 +75,7 @@ const createMainPackageJson = (target, format) => (orig) => ({ }, './*': { import: { - types: `./*.d.mts`, + types: `./*.d.ts`, default: `./*.mjs`, }, require: { @@ -127,8 +127,8 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) => sideEffects: format === 'esm' ? false : undefined, // include "esm" settings for https://www.npmjs.com/package/esm if building scoped ESM target esm: format === `esm` ? { mode: `auto`, sourceMap: true } : undefined, - // set "types" (for TypeScript/VSCode) - types: format === 'umd' ? undefined : `${mainExport}.node.d.ts`, + // set "types" to "Arrow.dom" if building scoped UMD target, otherwise "Arrow.node" + types: format === 'umd' ? `${mainExport}.dom.d.ts`: `${mainExport}.node.d.ts`, } ) ); diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js index b5a4c3232dc76..ae5b389f7f8d4 100644 --- a/js/gulp/typescript-task.js +++ b/js/gulp/typescript-task.js @@ -61,10 +61,9 @@ function compileTypescript(out, tsconfigPath, tsconfigOverrides, writeSourcemaps ); const writeSources = observableFromStreams(tsProject.src(), gulp.dest(path.join(out, 'src'))); const writeDTypes = observableFromStreams(dts, sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }), gulp.dest(out)); - const mapFile = tsProject.options.module === tsc.ModuleKind.ES2015 ? esmMapFile : cjsMapFile; const writeJSArgs = writeSourcemaps ? [ js, - sourcemaps.write('./', { mapFile, includeContent: false, sourceRoot: './src' }), + sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }), gulp.dest(out) ] : [ js, @@ -73,6 +72,3 @@ function compileTypescript(out, tsconfigPath, tsconfigOverrides, writeSourcemaps const writeJS = observableFromStreams(...writeJSArgs); return ObservableForkJoin([writeSources, writeDTypes, writeJS]); } - -const cjsMapFile = (mapFilePath) => mapFilePath; -const esmMapFile = (mapFilePath) => mapFilePath.replace('.js.map', '.mjs.map');