Skip to content

Commit 61c17dc

Browse files
tsufievtsufiev
committed
feat: introduce new type aliases: Types.BYTES & Types.TEXT
First one properly handles byte-array data: it converts string values to Buffer before sending data to YDB and do not implicitly transform values received from YDB to a string via any encoding, leaving this choice to the client. Doing so allows to keep the old behavior for Types.STRING type and avoid breaking the conversion of STRING values already stored in YDB. Types.TEXT behaves the same way as Types.UTF8 and was added just to mirror a new type alias in YDB. What was wrong with Types.STRING before? The most basic layer of ydb-sdk (bundle.js got from protobufs) works with STRING and YSON values as Node.js Buffer type. To handle these values properly, the same transformation to/from Buffer should be applied on top of basic layer: 1. before sending the data to YDB, from string to Buffer (if string was used); 2. upon receiving data from YDB, from Buffer to string, in case the client expects the data to be string. To maintain backwards compatibility (1) is not done for Types.STRING and Types.YSON, so that the format of data already in YDB does not change. The type of transformation (2) Types.STRING and Types.YSON is utf8, as before.
1 parent 5b9a68c commit 61c17dc

File tree

3 files changed

+182
-36
lines changed

3 files changed

+182
-36
lines changed

‎src/__tests__/bytestring-identity.test.ts

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import Driver from '../driver';
2+
import {
3+
destroyDriver,
4+
initDriver,
5+
TABLE
6+
} from '../test-utils';
7+
import {Column, Session, TableDescription} from '../table';
8+
import {declareType, TypedData, Types} from "../types";
9+
import {withRetries} from "../retries";
10+
11+
12+
async function createTable(session: Session) {
13+
await session.dropTable(TABLE);
14+
await session.createTable(
15+
TABLE,
16+
new TableDescription()
17+
.withColumn(new Column(
18+
'id',
19+
Types.optional(Types.UINT64),
20+
))
21+
.withColumn(new Column(
22+
'field1',
23+
Types.optional(Types.STRING),
24+
))
25+
.withColumn(new Column(
26+
'field2',
27+
Types.optional(Types.STRING),
28+
))
29+
.withPrimaryKey('id')
30+
);
31+
}
32+
33+
export interface IRow {
34+
id: number;
35+
field1: string;
36+
field2: Buffer;
37+
}
38+
39+
class Row extends TypedData {
40+
@declareType(Types.UINT64)
41+
public id: number;
42+
43+
@declareType(Types.STRING)
44+
public field1: string;
45+
46+
@declareType(Types.BYTES)
47+
public field2: Buffer;
48+
49+
constructor(data: IRow) {
50+
super(data);
51+
this.id = data.id;
52+
this.field1 = data.field1;
53+
this.field2 = data.field2;
54+
}
55+
}
56+
57+
export async function fillTableWithData(session: Session, rows: Row[]) {
58+
const query = `
59+
DECLARE $data AS List<Struct<id: Uint64, field1: String, field2: String>>;
60+
61+
REPLACE INTO ${TABLE}
62+
SELECT * FROM AS_TABLE($data);`;
63+
64+
await withRetries(async () => {
65+
const preparedQuery = await session.prepareQuery(query);
66+
await session.executeQuery(preparedQuery, {
67+
'$data': Row.asTypedCollection(rows),
68+
});
69+
});
70+
}
71+
72+
describe('bytestring identity', () => {
73+
let driver: Driver;
74+
let actualRows: Row[];
75+
const initialRows = [
76+
new Row({id: 0, field1: 'zero', field2: Buffer.from('half')}),
77+
];
78+
79+
afterAll(async () => await destroyDriver(driver));
80+
81+
beforeAll(async () => {
82+
driver = await initDriver();
83+
await driver.tableClient.withSession(async (session) => {
84+
await createTable(session);
85+
await fillTableWithData(session, initialRows);
86+
87+
const {resultSets} = await session.executeQuery(`SELECT * FROM ${TABLE}`);
88+
actualRows = Row.createNativeObjects(resultSets[0]) as Row[];
89+
});
90+
});
91+
92+
it('Types.STRING does not keep the original string in write-read cycle', () => {
93+
expect(actualRows[0].field1).not.toEqual('zero');
94+
});
95+
96+
it('Types.BYTES keeps the original string in write-read cycle', () => {
97+
expect(actualRows[0].field2.toString()).toEqual('half');
98+
});
99+
});

‎src/__tests__/types.test.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,18 @@ describe('Types', () => {
7676
type: {typeId: Ydb.Type.PrimitiveTypeId.STRING},
7777
value: {bytesValue: 'foo'},
7878
});
79+
expect(TypedValues.bytes(Buffer.from('foo'))).toEqual({
80+
type: {typeId: Ydb.Type.PrimitiveTypeId.STRING},
81+
value: {bytesValue: Buffer.from('foo')},
82+
});
7983
expect(TypedValues.utf8('привет')).toEqual({
8084
type: {typeId: Ydb.Type.PrimitiveTypeId.UTF8},
8185
value: {textValue: 'привет'},
8286
});
87+
expect(TypedValues.text('привет')).toEqual({
88+
type: {typeId: Ydb.Type.PrimitiveTypeId.UTF8},
89+
value: {textValue: 'привет'},
90+
});
8391
expect(TypedValues.yson('<a=1>[3;%false]')).toEqual({
8492
type: {typeId: Ydb.Type.PrimitiveTypeId.YSON},
8593
value: {bytesValue: '<a=1>[3;%false]'},

‎src/types.ts

Lines changed: 75 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import 'reflect-metadata';
55
import {DateTime} from 'luxon';
66
import {uuidToNative, uuidToValue} from './uuid';
77
import {fromDecimalString, toDecimalString} from './decimal';
8+
import {emitWarning} from 'process';
89
import Type = Ydb.Type;
910
import IType = Ydb.IType;
1011
import IStructMember = Ydb.IStructMember;
@@ -17,7 +18,21 @@ import NullValue = google.protobuf.NullValue;
1718

1819
export const typeMetadataKey = Symbol('type');
1920

21+
const shownDeprecations = new Set();
22+
function warnDeprecation(message: string) {
23+
if (!shownDeprecations.has(message)) {
24+
shownDeprecations.add(message);
25+
emitWarning(message);
26+
}
27+
}
28+
2029
export function declareType(type: IType) {
30+
if (type === Types.STRING) {
31+
warnDeprecation(
32+
'Types.STRING type is deprecated and will be removed in the next major release. Please migrate ' +
33+
'to the newer type Types.BYTES which avoids implicit conversions between Buffer and string types.'
34+
);
35+
}
2136
return Reflect.metadata(typeMetadataKey, type);
2237
}
2338

@@ -48,7 +63,7 @@ export const primitiveTypeToValue: Record<number, string> = {
4863
[Type.PrimitiveTypeId.TZ_TIMESTAMP]: 'textValue',
4964
};
5065

51-
type primitive = boolean | string | number | Long | Date;
66+
type primitive = boolean | string | number | Long | Date | Buffer;
5267

5368
export type StructFields = Record<string, IType>;
5469

@@ -57,15 +72,17 @@ export class Types {
5772
static INT8: IType = {typeId: Ydb.Type.PrimitiveTypeId.INT8};
5873
static UINT8: IType = {typeId: Ydb.Type.PrimitiveTypeId.UINT8};
5974
static INT16: IType = {typeId: Ydb.Type.PrimitiveTypeId.INT16};
60-
static UINT16: IType = {typeId: Ydb.Type.PrimitiveTypeId.INT16};
75+
static UINT16: IType = {typeId: Ydb.Type.PrimitiveTypeId.UINT16};
6176
static INT32: IType = {typeId: Ydb.Type.PrimitiveTypeId.INT32};
6277
static UINT32: IType = {typeId: Ydb.Type.PrimitiveTypeId.UINT32};
6378
static INT64: IType = {typeId: Ydb.Type.PrimitiveTypeId.INT64};
6479
static UINT64: IType = {typeId: Ydb.Type.PrimitiveTypeId.UINT64};
6580
static FLOAT: IType = {typeId: Ydb.Type.PrimitiveTypeId.FLOAT};
6681
static DOUBLE: IType = {typeId: Ydb.Type.PrimitiveTypeId.DOUBLE};
6782
static STRING: IType = {typeId: Ydb.Type.PrimitiveTypeId.STRING};
83+
static BYTES: IType = {typeId: Ydb.Type.PrimitiveTypeId.STRING};
6884
static UTF8: IType = {typeId: Ydb.Type.PrimitiveTypeId.UTF8};
85+
static TEXT: IType = {typeId: Ydb.Type.PrimitiveTypeId.UTF8};
6986
static YSON: IType = {typeId: Ydb.Type.PrimitiveTypeId.YSON};
7087
static JSON: IType = {typeId: Ydb.Type.PrimitiveTypeId.JSON};
7188
static UUID: IType = {typeId: Ydb.Type.PrimitiveTypeId.UUID};
@@ -135,11 +152,10 @@ export class Types {
135152
}
136153

137154
export class TypedValues {
138-
private static primitive(type: Ydb.Type.PrimitiveTypeId, value: primitive): ITypedValue {
139-
const primitiveType = {typeId: type};
155+
private static primitive(type: IType, value: primitive): ITypedValue {
140156
return {
141-
type: primitiveType,
142-
value: typeToValue(primitiveType, value),
157+
type: type,
158+
value: typeToValue(type, value),
143159
};
144160
}
145161

@@ -158,103 +174,115 @@ export class TypedValues {
158174
}
159175

160176
static bool(value: boolean): ITypedValue {
161-
return TypedValues.primitive(Type.PrimitiveTypeId.BOOL, value);
177+
return TypedValues.primitive(Types.BOOL, value);
162178
}
163179

164180
static int8(value: number): ITypedValue {
165-
return TypedValues.primitive(Type.PrimitiveTypeId.INT8, value);
181+
return TypedValues.primitive(Types.INT8, value);
166182
}
167183

168184
static uint8(value: number): ITypedValue {
169-
return TypedValues.primitive(Type.PrimitiveTypeId.UINT8, value);
185+
return TypedValues.primitive(Types.UINT8, value);
170186
}
171187

172188
static int16(value: number): ITypedValue {
173-
return TypedValues.primitive(Type.PrimitiveTypeId.INT16, value);
189+
return TypedValues.primitive(Types.INT16, value);
174190
}
175191

176192
static uint16(value: number): ITypedValue {
177-
return TypedValues.primitive(Type.PrimitiveTypeId.UINT16, value);
193+
return TypedValues.primitive(Types.UINT16, value);
178194
}
179195

180196
static int32(value: number): ITypedValue {
181-
return TypedValues.primitive(Type.PrimitiveTypeId.INT32, value);
197+
return TypedValues.primitive(Types.INT32, value);
182198
}
183199

184200
static uint32(value: number): ITypedValue {
185-
return TypedValues.primitive(Type.PrimitiveTypeId.UINT32, value);
201+
return TypedValues.primitive(Types.UINT32, value);
186202
}
187203

188204
static int64(value: number | Long): ITypedValue {
189-
return TypedValues.primitive(Type.PrimitiveTypeId.INT64, value);
205+
return TypedValues.primitive(Types.INT64, value);
190206
}
191207

192208
static uint64(value: number | Long): ITypedValue {
193-
return TypedValues.primitive(Type.PrimitiveTypeId.UINT64, value);
209+
return TypedValues.primitive(Types.UINT64, value);
194210
}
195211

196212
static float(value: number): ITypedValue {
197-
return TypedValues.primitive(Type.PrimitiveTypeId.FLOAT, value);
213+
return TypedValues.primitive(Types.FLOAT, value);
198214
}
199215

200216
static double(value: number): ITypedValue {
201-
return TypedValues.primitive(Type.PrimitiveTypeId.DOUBLE, value);
217+
return TypedValues.primitive(Types.DOUBLE, value);
202218
}
203219

204220
static string(value: string): ITypedValue {
205-
return TypedValues.primitive(Type.PrimitiveTypeId.STRING, value);
221+
warnDeprecation(
222+
'string() helper is deprecated and will be removed in the next major release. Please migrate ' +
223+
'to the newer helper bytes() which avoids implicit conversions between Buffer and string types.'
224+
);
225+
return TypedValues.primitive(Types.STRING, value);
226+
}
227+
228+
static bytes(value: Buffer): ITypedValue {
229+
return TypedValues.primitive(Types.BYTES, value);
206230
}
207231

208232
static utf8(value: string): ITypedValue {
209-
return TypedValues.primitive(Type.PrimitiveTypeId.UTF8, value);
233+
return TypedValues.primitive(Types.UTF8, value);
234+
}
235+
236+
static text(value: string): ITypedValue {
237+
return TypedValues.primitive(Types.TEXT, value);
210238
}
211239

212240
static yson(value: string): ITypedValue {
213-
return TypedValues.primitive(Type.PrimitiveTypeId.YSON, value);
241+
return TypedValues.primitive(Types.YSON, value);
214242
}
215243

216244
static json(value: string): ITypedValue {
217-
return TypedValues.primitive(Type.PrimitiveTypeId.JSON, value);
245+
return TypedValues.primitive(Types.JSON, value);
218246
}
219247

220248
static uuid(value: string): ITypedValue {
221-
return TypedValues.primitive(Type.PrimitiveTypeId.UUID, value);
249+
return TypedValues.primitive(Types.UUID, value);
222250
}
223251

224252
static jsonDocument(value: string): ITypedValue {
225-
return TypedValues.primitive(Type.PrimitiveTypeId.JSON_DOCUMENT, value);
253+
return TypedValues.primitive(Types.JSON_DOCUMENT, value);
226254
}
227255

228256
static date(value: Date): ITypedValue {
229-
return TypedValues.primitive(Type.PrimitiveTypeId.DATE, value);
257+
return TypedValues.primitive(Types.DATE, value);
230258
}
231259

232260
static datetime(value: Date): ITypedValue {
233-
return TypedValues.primitive(Type.PrimitiveTypeId.DATETIME, value);
261+
return TypedValues.primitive(Types.DATETIME, value);
234262
}
235263

236264
static timestamp(value: Date): ITypedValue {
237-
return TypedValues.primitive(Type.PrimitiveTypeId.TIMESTAMP, value);
265+
return TypedValues.primitive(Types.TIMESTAMP, value);
238266
}
239267

240268
static interval(value: number): ITypedValue {
241-
return TypedValues.primitive(Type.PrimitiveTypeId.INTERVAL, value);
269+
return TypedValues.primitive(Types.INTERVAL, value);
242270
}
243271

244272
static tzDate(value: Date): ITypedValue {
245-
return TypedValues.primitive(Type.PrimitiveTypeId.TZ_DATE, value);
273+
return TypedValues.primitive(Types.TZ_DATE, value);
246274
}
247275

248276
static tzDatetime(value: Date): ITypedValue {
249-
return TypedValues.primitive(Type.PrimitiveTypeId.TZ_DATETIME, value);
277+
return TypedValues.primitive(Types.TZ_DATETIME, value);
250278
}
251279

252280
static tzTimestamp(value: Date): ITypedValue {
253-
return TypedValues.primitive(Type.PrimitiveTypeId.TZ_TIMESTAMP, value);
281+
return TypedValues.primitive(Types.TZ_TIMESTAMP, value);
254282
}
255283

256284
static dynumber(value: string): ITypedValue {
257-
return TypedValues.primitive(Type.PrimitiveTypeId.DYNUMBER, value);
285+
return TypedValues.primitive(Types.DYNUMBER, value);
258286
}
259287

260288
static optional(value: Ydb.ITypedValue): Ydb.ITypedValue {
@@ -345,7 +373,7 @@ const valueToNativeConverters: Record<string, (input: string|number) => any> = {
345373
'uint64Value': (input) => parseLong(input),
346374
'floatValue': (input) => Number(input),
347375
'doubleValue': (input) => Number(input),
348-
'bytesValue': (input) => Buffer.from(input as string, 'base64').toString(),
376+
'bytesValue': (input) => input,
349377
'textValue': (input) => input,
350378
'nullFlagValue': () => null,
351379
};
@@ -360,7 +388,7 @@ function convertYdbValueToNative(type: IType, value: IValue): any {
360388
throw new Error(`Unknown PrimitiveTypeId: ${type.typeId}`);
361389
}
362390
const input = (value as any)[label];
363-
return objectFromValue(type.typeId, valueToNativeConverters[label](input));
391+
return objectFromValue(type, valueToNativeConverters[label](input));
364392
} else if (type.decimalType) {
365393
const high128 = value.high_128 as number | Long;
366394
const low128 = value.low_128 as number | Long;
@@ -433,8 +461,15 @@ function convertYdbValueToNative(type: IType, value: IValue): any {
433461
}
434462
}
435463

436-
function objectFromValue(typeId: PrimitiveTypeId, value: unknown) {
464+
function objectFromValue(type: IType, value: unknown) {
465+
if (type === Types.BYTES) {
466+
return value as Buffer;
467+
}
468+
const {typeId} = type;
437469
switch (typeId) {
470+
case PrimitiveTypeId.YSON:
471+
case PrimitiveTypeId.STRING:
472+
return (value as Buffer).toString('utf8');
438473
case PrimitiveTypeId.DATE:
439474
return new Date((value as number) * 3600 * 1000 * 24);
440475
case PrimitiveTypeId.DATETIME:
@@ -452,7 +487,11 @@ function objectFromValue(typeId: PrimitiveTypeId, value: unknown) {
452487
}
453488
}
454489

455-
function preparePrimitiveValue(typeId: PrimitiveTypeId, value: any) {
490+
function preparePrimitiveValue(type: IType, value: any) {
491+
if (type === Types.BYTES) {
492+
return value instanceof Buffer ? value : Buffer.from(value);
493+
}
494+
const typeId = type.typeId;
456495
switch (typeId) {
457496
case PrimitiveTypeId.DATE:
458497
return Number(value) / 3600 / 1000 / 24;
@@ -484,7 +523,7 @@ function typeToValue(type: IType | null | undefined, value: any): IValue {
484523
}
485524
const valueLabel = primitiveTypeToValue[type.typeId];
486525
if (valueLabel) {
487-
return {[valueLabel]: preparePrimitiveValue(type.typeId, value)};
526+
return {[valueLabel]: preparePrimitiveValue(type, value)};
488527
} else {
489528
throw new Error(`Unknown PrimitiveTypeId: ${type.typeId}`);
490529
}

0 commit comments

Comments
 (0)