Skip to content

Commit

Permalink
Merge branch 'main' into dom/decimal-scale
Browse files Browse the repository at this point in the history
  • Loading branch information
@domoritz
domoritz committed Apr 18, 2024
2 parents 28ebb3a + be5f5b8 commit 84ff0b3
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 52 deletions.
22 changes: 11 additions & 11 deletions csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ private void CompareBinaryArrays<T>(BinaryArray actualArray)
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

if (_strictCompare)
{
Expand Down Expand Up @@ -294,7 +294,7 @@ private void CompareVariadicArrays<T>(BinaryViewArray actualArray)
Assert.Equal(expectedArray.Offset, actualArray.Offset);
}

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

Assert.True(expectedArray.Views.SequenceEqual(actualArray.Views));

Expand All @@ -318,7 +318,7 @@ private void CompareArrays(FixedSizeBinaryArray actualArray)
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

if (_strictCompare)
{
Expand Down Expand Up @@ -347,7 +347,7 @@ private void CompareArrays<T>(PrimitiveArray<T> actualArray)
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

if (_strictCompare)
{
Expand Down Expand Up @@ -379,7 +379,7 @@ private void CompareArrays(BooleanArray actualArray)
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

if (_strictCompare)
{
Expand All @@ -406,7 +406,7 @@ private void CompareArrays(ListArray actualArray)
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

if (_strictCompare)
{
Expand Down Expand Up @@ -442,7 +442,7 @@ private void CompareArrays(ListViewArray actualArray)
Assert.Equal(expectedArray.Length, actualArray.Length);
Assert.Equal(expectedArray.NullCount, actualArray.NullCount);

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

if (_strictCompare)
{
Expand Down Expand Up @@ -475,15 +475,15 @@ private void CompareArrays(FixedSizeListArray actualArray)
Assert.Equal(expectedArray.Offset, actualArray.Offset);
}

CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer);
CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, expectedArray.Offset, actualArray.NullBitmapBuffer, actualArray.Offset);

var listSize = ((FixedSizeListType)expectedArray.Data.DataType).ListSize;
var expectedValuesSlice = ArrowArrayFactory.Slice(
expectedArray.Values, expectedArray.Offset * listSize, expectedArray.Length * listSize);
actualArray.Values.Accept(new ArrayComparer(expectedValuesSlice, _strictCompare));
}

private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer expectedValidityBuffer, int expectedBufferOffset, ArrowBuffer actualValidityBuffer)
private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer expectedValidityBuffer, int expectedBufferOffset, ArrowBuffer actualValidityBuffer, int actualBufferOffset)
{
if (_strictCompare)
{
Expand All @@ -493,7 +493,7 @@ private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer e
{
Assert.True(nullCount == 0 || arrayLength == 0);
}
else if (expectedBufferOffset % 8 == 0)
else if (expectedBufferOffset % 8 == 0 && expectedBufferOffset == actualBufferOffset)
{
int validityBitmapByteCount = BitUtility.ByteCount(arrayLength);
int byteOffset = BitUtility.ByteCount(expectedBufferOffset);
Expand Down Expand Up @@ -524,7 +524,7 @@ private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer e
for (int i = 0; i < arrayLength; i++)
{
Assert.True(
BitUtility.GetBit(expectedSpan, expectedBufferOffset + i) == BitUtility.GetBit(actualSpan, i),
BitUtility.GetBit(expectedSpan, expectedBufferOffset + i) == BitUtility.GetBit(actualSpan, actualBufferOffset + i),
string.Format("Bit at index {0}/{1} is not equal", i, arrayLength));
}
}
Expand Down
26 changes: 8 additions & 18 deletions js/.vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,14 @@
"version": "0.2.0",
"inputs": [
{
"type": "pickString",
"default": "src",
"type": "command",
"id": "TEST_TARGET",
"options": [
"src",
"apache-arrow",
"ts",
"es5.cjs",
"es5.esm",
"es5.umd",
"es2015.cjs",
"es2015.esm",
"es2015.umd",
"esnext.cjs",
"esnext.esm",
"esnext.umd",
],
"description": "The JS version + Module format combination to test (or src to test source files)",
"command": "shellCommand.execute",
"args": {
"cwd": "${workspaceFolder}",
"description": "The JS version + Module format combination to test (or src to test source files)",
"command": "echo \"src\napache-arrow\nts\nes5.cjs\nes5.esm\nes5.umd\nes2015.cjs\nes2015.esm\nes2015.umd\nesnext.cjs\nesnext.esm\nesnext.umd\""
}
},
{
"type": "command",
Expand Down Expand Up @@ -50,7 +40,7 @@
"command": "shellCommand.execute",
"args": {
"useSingleResult": "true",
"command": "case \"${input:TEST_TARGET}\" in *cjs | *umd | apache-arrow) echo '';; *) echo '--experimental-vm-modules';; esac"
"command": "case \"${input:TEST_TARGET}\" in *cjs | *umd) echo '--no-warnings';; *) echo '--experimental-vm-modules';; esac"
}
},
],
Expand Down
9 changes: 4 additions & 5 deletions js/gulp/arrow-task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,23 @@ import { pipeline } from 'stream/promises';

export const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) {
const out = targetDir(target);
const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`;
const dtsGlob = `${targetDir(`es2015`, `esm`)}/**/*.ts`;
const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`;
const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`;
const esnextUmdGlob = `${targetDir(`esnext`, `umd`)}/*.js`;
const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.map`;
const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js.map`;
const esmSourceMapsGlob = `${targetDir(`es2015`, `esm`)}/**/*.map`;
const es2015UmdSourceMapsGlob = `${targetDir(`es2015`, `umd`)}/*.map`;
const esnextUmdSourceMapsGlob = `${targetDir(`esnext`, `umd`)}/*.map`;
return ObservableForkJoin([
observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files
observableFromStreams(gulp.src(dtsGlob), gulpRename((p) => { p.extname = '.mts'; }), gulp.dest(out)), // copy d.ts files as esm
observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files
observableFromStreams(gulp.src(cjsSourceMapsGlob), gulp.dest(out)), // copy es2015 cjs sourcemaps
observableFromStreams(gulp.src(esmSourceMapsGlob), gulp.dest(out)), // copy es2015 esm sourcemaps
observableFromStreams(gulp.src(esmSourceMapsGlob), gulpRename((p) => { p.basename = p.basename.replace('.js', '.mjs'); }), gulpReplace(`.js"`, `.mjs"`), gulp.dest(out)), // copy es2015 esm sourcemaps
observableFromStreams(gulp.src(es2015UmdSourceMapsGlob), gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename
observableFromStreams(gulp.src(esnextUmdSourceMapsGlob), gulp.dest(out)), // copy esnext umd sourcemap files, but don't rename
observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulpReplace(`.js.map`, `.mjs.map`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
observableFromStreams(gulp.src(es2015UmdGlob), gulpRename((p) => { p.basename += `.es2015.min`; }), gulp.dest(out)), // copy es2015 umd files and add `.es2015.min`
observableFromStreams(gulp.src(esnextUmdGlob), gulpRename((p) => { p.basename += `.esnext.min`; }), gulp.dest(out)), // copy esnext umd files and add `.esnext.min`
]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false }));
Expand Down
68 changes: 60 additions & 8 deletions js/gulp/closure-task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,34 @@ import { targetDir, mainExport, esmRequire, gCCLanguageNames, publicModulePaths,

import fs from 'node:fs';
import gulp from 'gulp';
import path from 'node:path';
import Path from 'node:path';
import https from 'node:https';
import { mkdirp } from 'mkdirp';
import { PassThrough } from 'node:stream';
import sourcemaps from 'gulp-sourcemaps';
import { memoizeTask } from './memoize-task.js';
import { compileBinFiles } from './typescript-task.js';

import closureCompiler from 'google-closure-compiler';
const compiler = closureCompiler.gulp();

import { fileURLToPath } from 'url';
import { dirname } from 'path';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const closureCompilerVer = JSON.parse(fs.readFileSync(Path.join(__dirname, '..', 'package.json'))).devDependencies['google-closure-compiler'].split('.')[0];

export const closureTask = ((cache) => memoizeTask(cache, async function closure(target, format) {

if (shouldRunInChildProcess(target, format)) {
return spawnGulpCommandInChildProcess('compile', target, format);
}

const src = targetDir(target, `cls`);
const srcAbsolute = path.resolve(src);
const srcAbsolute = Path.resolve(src);
const out = targetDir(target, format);
const externs = path.join(`${out}/${mainExport}.externs.js`);
const entry_point = path.join(`${src}/${mainExport}.dom.cls.js`);
const externs = Path.join(`${out}/${mainExport}.externs.js`);
const entry_point = Path.join(`${src}/${mainExport}.dom.cls.js`);

const exportedImports = publicModulePaths(srcAbsolute).reduce((entries, publicModulePath) => [
...entries, {
Expand All @@ -54,12 +62,42 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure
fs.promises.writeFile(entry_point, generateUMDExportAssignment(srcAbsolute, exportedImports))
]);

return await Promise.all([
const closeCompilerPolyfills = [];

await Promise.all([
runClosureCompileAsObservable().toPromise(),
compileBinFiles(target, format).toPromise(),
observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)), // copy .d.ts files
compileBinFiles(target, format).toPromise().then(() => Promise.all([
observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)).toPromise(), // copy .d.ts files,
observableFromStreams(gulp.src(`${src}/**/*.d.ts.map`), gulp.dest(out)).toPromise(), // copy .d.ts.map files,
observableFromStreams(gulp.src(`${src}/src/**/*`), gulp.dest(`${out}/src`)).toPromise(), // copy TS source files,
]))
]);

// Download the closure compiler polyfill sources for sourcemaps
await Promise.all(closeCompilerPolyfills.map(async (path) => {

await fs.promises.mkdir(
Path.join(out, Path.parse(path).dir),
{ recursive: true, mode: 0o755 }
);

const res = new PassThrough();
const req = https.request(
new URL(`https://raw.githubusercontent.com/google/closure-compiler/v${closureCompilerVer}/${path}`),
(res_) => {
if (res_.statusCode === 200) {
res_.pipe(res);
} else {
res.end();
}
}
);

req.on('error', (e) => res.emit('error', e)).end();

return observableFromStreams(res, fs.createWriteStream(Path.join(out, path))).toPromise();
}));

function runClosureCompileAsObservable() {
return observableFromStreams(
gulp.src([
Expand All @@ -72,8 +110,22 @@ export const closureTask = ((cache) => memoizeTask(cache, async function closure
compiler(createClosureArgs(entry_point, externs, target), {
platform: ['native', 'java', 'javascript']
}),
sourcemaps.mapSources((path) => {
if (path.indexOf(`${src}/`) === 0) {
return path.slice(`${src}/`.length);
}
if (path.includes('com/google')) {
closeCompilerPolyfills.push(path);
return path.slice(`src/`.length);
}
return path;
}),
// rename the sourcemaps from *.js.map files to *.min.js.map
sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
sourcemaps.write(`./`, {
sourceRoot: './src',
includeContent: false,
mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`),
}),
gulp.dest(out)
);
}
Expand Down
10 changes: 5 additions & 5 deletions js/gulp/package-task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
'.': {
node: {
import: {
types: `./${mainExport}.node.d.mts`,
types: `./${mainExport}.node.d.ts`,
default: `./${mainExport}.node.mjs`,
},
require: {
Expand All @@ -65,7 +65,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
},
},
import: {
types: `./${mainExport}.dom.d.mts`,
types: `./${mainExport}.dom.d.ts`,
default: `./${mainExport}.dom.mjs`,
},
require: {
Expand All @@ -75,7 +75,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
},
'./*': {
import: {
types: `./*.d.mts`,
types: `./*.d.ts`,
default: `./*.mjs`,
},
require: {
Expand Down Expand Up @@ -127,8 +127,8 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
sideEffects: format === 'esm' ? false : undefined,
// include "esm" settings for https://www.npmjs.com/package/esm if building scoped ESM target
esm: format === `esm` ? { mode: `auto`, sourceMap: true } : undefined,
// set "types" (for TypeScript/VSCode)
types: format === 'umd' ? undefined : `${mainExport}.node.d.ts`,
// set "types" to "Arrow.dom" if building scoped UMD target, otherwise "Arrow.node"
types: format === 'umd' ? `${mainExport}.dom.d.ts`: `${mainExport}.node.d.ts`,
}
)
);
6 changes: 1 addition & 5 deletions js/gulp/typescript-task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,9 @@ function compileTypescript(out, tsconfigPath, tsconfigOverrides, writeSourcemaps
);
const writeSources = observableFromStreams(tsProject.src(), gulp.dest(path.join(out, 'src')));
const writeDTypes = observableFromStreams(dts, sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }), gulp.dest(out));
const mapFile = tsProject.options.module === tsc.ModuleKind.ES2015 ? esmMapFile : cjsMapFile;
const writeJSArgs = writeSourcemaps ? [
js,
sourcemaps.write('./', { mapFile, includeContent: false, sourceRoot: './src' }),
sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }),
gulp.dest(out)
] : [
js,
Expand All @@ -73,6 +72,3 @@ function compileTypescript(out, tsconfigPath, tsconfigOverrides, writeSourcemaps
const writeJS = observableFromStreams(...writeJSArgs);
return ObservableForkJoin([writeSources, writeDTypes, writeJS]);
}

const cjsMapFile = (mapFilePath) => mapFilePath;
const esmMapFile = (mapFilePath) => mapFilePath.replace('.js.map', '.mjs.map');

0 comments on commit 84ff0b3

Please sign in to comment.