Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(sql) fix support for binary numeric values #17245

Merged
merged 2 commits into from
Feb 11, 2025
Merged

fix(sql) fix support for binary numeric values #17245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e37c02e
fix support for binary numeric values
cirospaciari Feb 11, 2025
51ce0a1
Merge branch 'main' into ciro/fix-numeric
cirospaciari Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/bun.js/bindings/ErrorCode.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ const errors: ErrorCodeMapping = [
["ERR_POSTGRES_UNSUPPORTED_BYTEA_FORMAT", TypeError, "PostgresError"],
["ERR_POSTGRES_UNSUPPORTED_ARRAY_FORMAT", TypeError, "PostgresError"],
["ERR_POSTGRES_UNSUPPORTED_INTEGER_SIZE", TypeError, "PostgresError"],
["ERR_POSTGRES_UNSUPPORTED_NUMERIC_FORMAT", TypeError, "PostgresError"],
["ERR_POSTGRES_IDLE_TIMEOUT", Error, "PostgresError"],
["ERR_POSTGRES_CONNECTION_TIMEOUT", Error, "PostgresError"],
["ERR_POSTGRES_LIFETIME_TIMEOUT", Error, "PostgresError"],
Expand Down
121 changes: 119 additions & 2 deletions src/sql/postgres.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ pub const AnyPostgresError = error{
UnsupportedByteaFormat,
UnsupportedIntegerSize,
UnsupportedArrayFormat,
UnsupportedNumericFormat,
};

pub fn postgresErrorToJS(globalObject: *JSC.JSGlobalObject, message: ?[]const u8, err: AnyPostgresError) JSValue {
Expand Down Expand Up @@ -75,6 +76,7 @@ pub fn postgresErrorToJS(globalObject: *JSC.JSGlobalObject, message: ?[]const u8
error.UnsupportedByteaFormat => JSC.Error.ERR_POSTGRES_UNSUPPORTED_BYTEA_FORMAT,
error.UnsupportedArrayFormat => JSC.Error.ERR_POSTGRES_UNSUPPORTED_ARRAY_FORMAT,
error.UnsupportedIntegerSize => JSC.Error.ERR_POSTGRES_UNSUPPORTED_INTEGER_SIZE,
error.UnsupportedNumericFormat => JSC.Error.ERR_POSTGRES_UNSUPPORTED_NUMERIC_FORMAT,
error.JSError => {
return globalObject.takeException(error.JSError);
},
Expand Down Expand Up @@ -2522,9 +2524,9 @@ pub const PostgresSQLConnection = struct {
} else {
// the only escape sequency possible here is \b
if (bun.strings.eqlComptime(element, "\\b")) {
try array.append(bun.default_allocator, DataCell{ .tag = .string, .value = .{ .string = bun.String.static("\x08").value.WTFStringImpl }, .free_value = 1 });
try array.append(bun.default_allocator, DataCell{ .tag = .string, .value = .{ .string = bun.String.createUTF8("\x08").value.WTFStringImpl }, .free_value = 1 });
} else {
try array.append(bun.default_allocator, DataCell{ .tag = .string, .value = .{ .string = if (element.len > 0) bun.String.createUTF8(element).value.WTFStringImpl else null }, .free_value = 1 });
try array.append(bun.default_allocator, DataCell{ .tag = .string, .value = .{ .string = if (element.len > 0) bun.String.createUTF8(element).value.WTFStringImpl else null }, .free_value = 0 });
}
}
slice = trySlice(slice, current_idx);
Expand Down Expand Up @@ -2834,6 +2836,23 @@ pub const PostgresSQLConnection = struct {
return DataCell{ .tag = .float8, .value = .{ .float8 = float4 } };
}
},
.numeric => {
if (binary) {
// this is probrably good enough for most cases
var stack_buffer = std.heap.stackFallback(1024, bun.default_allocator);
const allocator = stack_buffer.get();
var numeric_buffer = std.ArrayList(u8).fromOwnedSlice(allocator, &stack_buffer.buffer);
numeric_buffer.items.len = 0;
defer numeric_buffer.deinit();

// if is binary format lets display as a string because JS cant handle it in a safe way
const result = parseBinaryNumeric(bytes, &numeric_buffer) catch return error.UnsupportedNumericFormat;
return DataCell{ .tag = .string, .value = .{ .string = bun.String.createUTF8(result.slice()).value.WTFStringImpl }, .free_value = 1 };
} else {
// nice text is actually what we want here
return DataCell{ .tag = .string, .value = .{ .string = if (bytes.len > 0) String.createUTF8(bytes).value.WTFStringImpl else null }, .free_value = 1 };
}
},
.jsonb, .json => {
return DataCell{ .tag = .json, .value = .{ .json = if (bytes.len > 0) String.createUTF8(bytes).value.WTFStringImpl else null }, .free_value = 1 };
},
Expand Down Expand Up @@ -2951,6 +2970,104 @@ pub const PostgresSQLConnection = struct {
fn pg_ntoh32(x: anytype) u32 {
return pg_ntoT(32, x);
}
const PGNummericString = union(enum) {
static: [:0]const u8,
dynamic: []const u8,

pub fn slice(this: PGNummericString) []const u8 {
return switch (this) {
.static => |value| value,
.dynamic => |value| value,
};
}
};

fn parseBinaryNumeric(input: []const u8, result: *std.ArrayList(u8)) !PGNummericString {
// Reference: https://github.com/postgres/postgres/blob/50e6eb731d98ab6d0e625a0b87fb327b172bbebd/src/backend/utils/adt/numeric.c#L7612-L7740
if (input.len < 8) return error.InvalidBuffer;
var fixed_buffer = std.io.fixedBufferStream(input);
var reader = fixed_buffer.reader();

// Read header values using big-endian
const ndigits = try reader.readInt(i16, .big);
const weight = try reader.readInt(i16, .big);
const sign = try reader.readInt(u16, .big);
const dscale = try reader.readInt(i16, .big);

// Handle special cases
switch (sign) {
0xC000 => return PGNummericString{ .static = "NaN" },
0xD000 => return PGNummericString{ .static = "Infinity" },
0xF000 => return PGNummericString{ .static = "-Infinity" },
0x4000, 0x0000 => {},
else => return error.InvalidSign,
}

if (ndigits == 0) {
return PGNummericString{ .static = "0" };
}

// Add negative sign if needed
if (sign == 0x4000) {
try result.append('-');
}

// Calculate decimal point position
var decimal_pos: i32 = @as(i32, weight + 1) * 4;
if (decimal_pos <= 0) {
decimal_pos = 1;
}
// Output all digits before the decimal point

var scale_start: i32 = 0;
if (weight < 0) {
try result.append('0');
scale_start = @as(i32, @intCast(weight)) + 1;
} else {
var idx: usize = 0;
var first_non_zero = false;

while (idx <= weight) : (idx += 1) {
const digit = if (idx < ndigits) try reader.readInt(u16, .big) else 0;
var digit_str: [4]u8 = undefined;
const digit_len = std.fmt.formatIntBuf(&digit_str, digit, 10, .lower, .{ .width = 4, .fill = '0' });
if (!first_non_zero) {
//In the first digit, suppress extra leading decimal zeroes
var start_idx: usize = 0;
while (start_idx < digit_len and digit_str[start_idx] == '0') : (start_idx += 1) {}
if (start_idx == digit_len) continue;
const digit_slice = digit_str[start_idx..digit_len];
try result.appendSlice(digit_slice);
first_non_zero = true;
} else {
try result.appendSlice(digit_str[0..digit_len]);
}
}
}
// If requested, output a decimal point and all the digits that follow it.
// We initially put out a multiple of 4 digits, then truncate if needed.
if (dscale > 0) {
try result.append('.');
// negative scale means we need to add zeros before the decimal point
// greater than ndigits means we need to add zeros after the decimal point
var idx: isize = scale_start;
const end: usize = result.items.len + @as(usize, @intCast(dscale));
while (idx < dscale) : (idx += 4) {
if (idx >= 0 and idx < ndigits) {
const digit = reader.readInt(u16, .big) catch 0;
var digit_str: [4]u8 = undefined;
const digit_len = std.fmt.formatIntBuf(&digit_str, digit, 10, .lower, .{ .width = 4, .fill = '0' });
try result.appendSlice(digit_str[0..digit_len]);
} else {
try result.appendSlice("0000");
}
}
if (result.items.len > end) {
result.items.len = end;
}
}
return PGNummericString{ .dynamic = result.items };
}

pub fn parseBinary(comptime tag: types.Tag, comptime ReturnType: type, bytes: []const u8) AnyPostgresError!ReturnType {
switch (comptime tag) {
Expand Down
205 changes: 205 additions & 0 deletions test/js/sql/sql.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10577,4 +10577,209 @@ if (isDockerEnabled()) {
expect(result[0].null_array).toBeNull();
});
});

describe("numeric", () => {
test("handles standard decimal numbers", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(10,4))`;

const body = [
{ area: "D", price: "0.00001" }, // should collapse to 0
{ area: "D", price: "0.0001" },
{ area: "D", price: "0.0010" },
{ area: "D", price: "0.0100" },
{ area: "D", price: "0.1000" },
{ area: "D", price: "1.0000" },
{ area: "D", price: "10.0000" },
{ area: "D", price: "100.0000" },
{ area: "D", price: "1000.0000" },
{ area: "D", price: "10000.0000" },
{ area: "D", price: "100000.0000" },

{ area: "D", price: "1.1234" },
{ area: "D", price: "10.1234" },
{ area: "D", price: "100.1234" },
{ area: "D", price: "1000.1234" },
{ area: "D", price: "10000.1234" },
{ area: "D", price: "100000.1234" },

{ area: "D", price: "1.1234" },
{ area: "D", price: "10.1234" },
{ area: "D", price: "101.1234" },
{ area: "D", price: "1010.1234" },
{ area: "D", price: "10100.1234" },
{ area: "D", price: "101000.1234" },

{ area: "D", price: "999999.9999" }, // limit of NUMERIC(10,4)

// negative numbers
{ area: "D", price: "-0.00001" }, // should collapse to 0
{ area: "D", price: "-0.0001" },
{ area: "D", price: "-0.0010" },
{ area: "D", price: "-0.0100" },
{ area: "D", price: "-0.1000" },
{ area: "D", price: "-1.0000" },
{ area: "D", price: "-10.0000" },
{ area: "D", price: "-100.0000" },
{ area: "D", price: "-1000.0000" },
{ area: "D", price: "-10000.0000" },
{ area: "D", price: "-100000.0000" },

{ area: "D", price: "-1.1234" },
{ area: "D", price: "-10.1234" },
{ area: "D", price: "-100.1234" },
{ area: "D", price: "-1000.1234" },
{ area: "D", price: "-10000.1234" },
{ area: "D", price: "-100000.1234" },

{ area: "D", price: "-1.1234" },
{ area: "D", price: "-10.1234" },
{ area: "D", price: "-101.1234" },
{ area: "D", price: "-1010.1234" },
{ area: "D", price: "-10100.1234" },
{ area: "D", price: "-101000.1234" },

{ area: "D", price: "-999999.9999" }, // limit of NUMERIC(10,4)

// NaN
{ area: "D", price: "NaN" },
];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
expect(results[0].price).toEqual("0");
expect(results[1].price).toEqual("0.0001");
expect(results[2].price).toEqual("0.0010");
expect(results[3].price).toEqual("0.0100");
expect(results[4].price).toEqual("0.1000");
expect(results[5].price).toEqual("1.0000");
expect(results[6].price).toEqual("10.0000");
expect(results[7].price).toEqual("100.0000");
expect(results[8].price).toEqual("1000.0000");
expect(results[9].price).toEqual("10000.0000");
expect(results[10].price).toEqual("100000.0000");

expect(results[11].price).toEqual("1.1234");
expect(results[12].price).toEqual("10.1234");
expect(results[13].price).toEqual("100.1234");
expect(results[14].price).toEqual("1000.1234");
expect(results[15].price).toEqual("10000.1234");
expect(results[16].price).toEqual("100000.1234");

expect(results[17].price).toEqual("1.1234");
expect(results[18].price).toEqual("10.1234");
expect(results[19].price).toEqual("101.1234");
expect(results[20].price).toEqual("1010.1234");
expect(results[21].price).toEqual("10100.1234");
expect(results[22].price).toEqual("101000.1234");

expect(results[23].price).toEqual("999999.9999");

// negative numbers
expect(results[24].price).toEqual("0");
expect(results[25].price).toEqual("-0.0001");
expect(results[26].price).toEqual("-0.0010");
expect(results[27].price).toEqual("-0.0100");
expect(results[28].price).toEqual("-0.1000");
expect(results[29].price).toEqual("-1.0000");
expect(results[30].price).toEqual("-10.0000");
expect(results[31].price).toEqual("-100.0000");
expect(results[32].price).toEqual("-1000.0000");
expect(results[33].price).toEqual("-10000.0000");
expect(results[34].price).toEqual("-100000.0000");

expect(results[35].price).toEqual("-1.1234");
expect(results[36].price).toEqual("-10.1234");
expect(results[37].price).toEqual("-100.1234");
expect(results[38].price).toEqual("-1000.1234");
expect(results[39].price).toEqual("-10000.1234");
expect(results[40].price).toEqual("-100000.1234");

expect(results[41].price).toEqual("-1.1234");
expect(results[42].price).toEqual("-10.1234");
expect(results[43].price).toEqual("-101.1234");
expect(results[44].price).toEqual("-1010.1234");
expect(results[45].price).toEqual("-10100.1234");
expect(results[46].price).toEqual("-101000.1234");

expect(results[47].price).toEqual("-999999.9999");

expect(results[48].price).toEqual("NaN");
});
test("handle different scales", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(20,10))`;
const body = [{ area: "D", price: "1010001010.1234" }];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
expect(results[0].price).toEqual("1010001010.1234000000");
});
test("handles leading zeros", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(10,4))`;
const body = [
{ area: "A", price: "00001.00045" }, // should collapse to 1.0005
{ area: "B", price: "0000.12345" }, // should collapse to 0.1235
];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
expect(results[0].price).toBe("1.0005");
expect(results[1].price).toBe("0.1235");
});

test("handles numbers at scale boundaries", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(10,4))`;
const body = [
{ area: "C", price: "999999.9999" }, // Max for NUMERIC(10,4)
{ area: "D", price: "0.0001" }, // Min positive for 4 decimals
];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
expect(results[0].price).toBe("999999.9999");
expect(results[1].price).toBe("0.0001");
});

test("handles zero values", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(10,4))`;
const body = [
{ area: "E", price: "0" },
{ area: "F", price: "0.0000" },
{ area: "G", price: "00000.0000" },
];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
results.forEach(row => {
expect(row.price).toBe("0");
});
});

test("handles negative numbers", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(10,4))`;
const body = [
{ area: "H", price: "-1.2345" },
{ area: "I", price: "-0.0001" },
{ area: "J", price: "-9999.9999" },
];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
expect(results[0].price).toBe("-1.2345");
expect(results[1].price).toBe("-0.0001");
expect(results[2].price).toBe("-9999.9999");
});

test("handles scientific notation", async () => {
await using sql = postgres({ ...options, max: 1 });
const random_name = "test_" + randomUUIDv7("hex").replaceAll("-", "");
await sql`CREATE TEMPORARY TABLE ${sql(random_name)} (area text, price NUMERIC(10,4))`;
const body = [
{ area: "O", price: "1.2345e1" }, // 12.345
{ area: "P", price: "1.2345e-2" }, // 0.012345
];
const results = await sql`INSERT INTO ${sql(random_name)} ${sql(body)} RETURNING *`;
expect(results[0].price).toBe("12.3450");
expect(results[1].price).toBe("0.0123");
});
});
}