Skip to content
/ qp-n8n Public
forked from n8n-io/n8n

Commit

Permalink
test: Add expression transform tests (n8n-io#5497) (no-changelog)
Browse files Browse the repository at this point in the history 
test: add expression transform tests
  • Loading branch information
@valya @sunilrr
valya authored and sunilrr committed Apr 24, 2023
1 parent 84c0b24 commit 5251c60
Show file tree
Hide file tree
Showing 4 changed files with 278 additions and 69 deletions.
49 changes: 4 additions & 45 deletions packages/workflow/src/Expression.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,14 @@ import type {
NodeParameterValueType,
WorkflowExecuteMode,
} from './Interfaces';
import { ExpressionError, ExpressionExtensionError } from './ExpressionError';
import { ExpressionError } from './ExpressionError';
import { WorkflowDataProxy } from './WorkflowDataProxy';
import type { Workflow } from './Workflow';

// eslint-disable-next-line import/no-cycle
import { extend, extendOptional, hasExpressionExtension, hasNativeMethod } from './Extensions';
import type { ExpressionChunk, ExpressionCode } from './Extensions/ExpressionParser';
import { joinExpression, splitExpression } from './Extensions/ExpressionParser';
import { extendTransform } from './Extensions/ExpressionExtension';
import { extend, extendOptional } from './Extensions';
import { extendedFunctions } from './Extensions/ExtendedFunctions';
import { extendSyntax } from './Extensions/ExpressionExtension';

// Set it to use double curly brackets instead of single ones
tmpl.brackets.set('{{ }}');
Expand Down Expand Up @@ -292,7 +290,7 @@ export class Expression {
}

// Execute the expression
const extendedExpression = this.extendSyntax(parameterValue);
const extendedExpression = extendSyntax(parameterValue);
const returnValue = this.renderExpression(extendedExpression, data);
if (typeof returnValue === 'function') {
if (returnValue.name === '$') throw new Error('invalid syntax');
Expand Down Expand Up @@ -358,45 +356,6 @@ export class Expression {
return null;
}

extendSyntax(bracketedExpression: string): string {
const chunks = splitExpression(bracketedExpression);

const codeChunks = chunks
.filter((c) => c.type === 'code')
.map((c) => c.text.replace(/("|').*?("|')/, '').trim());

if (!codeChunks.some(hasExpressionExtension) || hasNativeMethod(bracketedExpression))
return bracketedExpression;

const extendedChunks = chunks.map((chunk): ExpressionChunk => {
if (chunk.type === 'code') {
const output = extendTransform(chunk.text);

// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (!output?.code) {
throw new ExpressionExtensionError('invalid syntax');
}

let text = output.code;

// We need to cut off any trailing semicolons. These cause issues
// with certain types of expression and cause the whole expression
// to fail.
if (text.trim().endsWith(';')) {
text = text.trim().slice(0, -1);
}

return {
...chunk,
text,
} as ExpressionCode;
}
return chunk;
});

return joinExpression(extendedChunks);
}

/**
* Resolves value of parameter. But does not work for workflow-data.
*
Expand Down
136 changes: 122 additions & 14 deletions packages/workflow/src/Extensions/ExpressionExtension.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { DateTime } from 'luxon';
import { ExpressionExtensionError } from '../ExpressionError';
import { parse, visit, types, print } from 'recast';
import { getOption } from 'recast/lib/util';
import type { Config as EsprimaConfig } from 'esprima-next';
import { parse as esprimaParse } from 'esprima-next';

import { arrayExtensions } from './ArrayExtensions';
Expand All @@ -12,6 +13,9 @@ import { stringExtensions } from './StringExtensions';
import { objectExtensions } from './ObjectExtensions';
import type { ExpressionKind } from 'ast-types/gen/kinds';

import type { ExpressionChunk, ExpressionCode } from './ExpressionParser';
import { joinExpression, splitExpression } from './ExpressionParser';

const EXPRESSION_EXTENDER = 'extend';
const EXPRESSION_EXTENDER_OPTIONAL = 'extendOptional';

Expand Down Expand Up @@ -89,17 +93,12 @@ function parseWithEsprimaNext(source: string, options?: any): any {
loc: true,
locations: true,
comment: true,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
range: getOption(options, 'range', false),
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
tolerant: getOption(options, 'tolerant', true),
range: getOption(options, 'range', false) as boolean,
tolerant: getOption(options, 'tolerant', true) as boolean,
tokens: true,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
jsx: getOption(options, 'jsx', false),
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
sourceType: getOption(options, 'sourceType', 'module'),
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any);
jsx: getOption(options, 'jsx', false) as boolean,
sourceType: getOption(options, 'sourceType', 'module') as string,
} as EsprimaConfig);

return ast;
}
Expand All @@ -124,9 +123,8 @@ export const extendTransform = (expression: string): { code: string } | undefine

let currentChain = 1;

// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
// Polyfill optional chaining
visit(ast, {
// Polyfill optional chaining
visitChainExpression(path) {
this.traverse(path);
const chainNumber = currentChain;
Expand All @@ -138,6 +136,8 @@ export const extendTransform = (expression: string): { code: string } | undefine
// @ts-ignore
typeof window !== 'object' ? 'global' : 'window',
);
// We want to define all of our commonly used identifiers and member
// expressions now so we don't have to create multiple instances
const undefinedIdentifier = types.builders.identifier('undefined');
const cancelIdentifier = types.builders.identifier(`chainCancelToken${chainNumber}`);
const valueIdentifier = types.builders.identifier(`chainValue${chainNumber}`);
Expand All @@ -152,6 +152,8 @@ export const extendTransform = (expression: string): { code: string } | undefine

const patchedStack: ExpressionKind[] = [];

// This builds the cancel check. This lets us slide to the end of the expression
// if it's undefined/null at any of the optional points of the chain.
const buildCancelCheckWrapper = (node: ExpressionKind): ExpressionKind => {
return types.builders.conditionalExpression(
types.builders.binaryExpression(
Expand All @@ -164,10 +166,17 @@ export const extendTransform = (expression: string): { code: string } | undefine
);
};

// This is just a quick small wrapper to create the assignment expression
// for the running value.
const buildValueAssignWrapper = (node: ExpressionKind): ExpressionKind => {
return types.builders.assignmentExpression('=', valueMemberExpression, node);
};

// This builds what actually does the comparison. It wraps the current
// chunk of the expression with a nullish coalescing operator that returns
// undefined if it's null or undefined. We do this because optional chains
// always return undefined if they fail part way, even if the value they
// fail on is null.
const buildOptionalWrapper = (node: ExpressionKind): ExpressionKind => {
return types.builders.binaryExpression(
'===',
Expand All @@ -180,6 +189,7 @@ export const extendTransform = (expression: string): { code: string } | undefine
);
};

// Another small wrapper, but for assigning to the cancel token this time.
const buildCancelAssignWrapper = (node: ExpressionKind): ExpressionKind => {
return types.builders.assignmentExpression('=', cancelMemberExpression, node);
};
Expand All @@ -189,6 +199,9 @@ export const extendTransform = (expression: string): { code: string } | undefine
let patchTop: ExpressionKind | null = null;
let wrapNextTopInOptionalExtend = false;

// This patches the previous node to use our current one as it's left hand value.
// It takes `window.chainValue1.test1` and `window.chainValue1.test2` and turns it
// into `window.chainValue1.test2.test1`.
const updatePatch = (toPatch: ExpressionKind, node: ExpressionKind) => {
if (toPatch.type === 'MemberExpression' || toPatch.type === 'OptionalMemberExpression') {
toPatch.object = node;
Expand All @@ -200,14 +213,26 @@ export const extendTransform = (expression: string): { code: string } | undefine
}
};

// This loop walks down an optional chain from the top. This will walk
// from right to left through an optional chain. We keep track of our current
// top of the chain (furthest right) and create a chain below it. This chain
// contains all of the (member and call) expressions that we need. These are
// patched versions that reference our current chain value. We then push this
// chain onto a stack when we hit an optional point in our chain.
while (true) {
// This should only ever be these types but you can optional chain on
// JSX nodes, which we don't support.
if (
currentNode.type === 'MemberExpression' ||
currentNode.type === 'OptionalMemberExpression' ||
currentNode.type === 'CallExpression' ||
currentNode.type === 'OptionalCallExpression'
) {
let patchNode: ExpressionKind;
// Here we take the current node and extract the parts we actually care
// about.
// In the case of a member expression we take the property it's trying to
// access and make the object it's accessing be our chain value.
if (
currentNode.type === 'MemberExpression' ||
currentNode.type === 'OptionalMemberExpression'
Expand All @@ -216,23 +241,31 @@ export const extendTransform = (expression: string): { code: string } | undefine
valueMemberExpression,
currentNode.property,
);
// In the case of a call expression we take the arguments and make the
// callee our chain value.
} else {
patchNode = types.builders.callExpression(
valueMemberExpression,
currentNode.arguments,
);
}

// If we have a previous node we patch it here.
if (currentPatch) {
updatePatch(currentPatch, patchNode);
}

// If we have no top patch (first run, or just pushed onto the stack) we
// note it here.
if (!patchTop) {
patchTop = patchNode;
}

currentPatch = patchNode;

// This is an optional in our chain. In here we'll push the node onto the
// stack. We also do a polyfill if the top of the stack is function call
// that might be a extended function.
if (currentNode.optional) {
// Implement polyfill described below
if (wrapNextTopInOptionalExtend) {
Expand Down Expand Up @@ -268,6 +301,7 @@ export const extendTransform = (expression: string): { code: string } | undefine
}
}

// Finally we get the next point AST to walk down.
if (
currentNode.type === 'MemberExpression' ||
currentNode.type === 'OptionalMemberExpression'
Expand All @@ -277,6 +311,8 @@ export const extendTransform = (expression: string): { code: string } | undefine
currentNode = currentNode.callee;
}
} else {
// We update the final patch to point to the first part of the optional chain
// which is probably an identifier for an object.
if (currentPatch) {
updatePatch(currentPatch, currentNode);
if (!patchTop) {
Expand All @@ -298,6 +334,7 @@ export const extendTransform = (expression: string): { code: string } | undefine
}
}

// Push the first part of our chain to stack.
if (patchTop) {
patchedStack.push(patchTop);
} else {
Expand All @@ -307,28 +344,42 @@ export const extendTransform = (expression: string): { code: string } | undefine
}
}

// Since we're working from right to left we need to flip the stack
// for the correct order of operations
patchedStack.reverse();

// Walk the node stack and wrap all our expressions in cancel/assignment
// wrappers.
for (let i = 0; i < patchedStack.length; i++) {
let node = patchedStack[i];

// We don't wrap the last expression in an assignment wrapper because
// it's going to be returned anyway. We just wrap it in a cancel check
// wrapper.
if (i !== patchedStack.length - 1) {
node = buildCancelAssignWrapper(buildOptionalWrapper(node));
}

// Don't wrap the first part in a cancel wrapper because the cancel
// token will always be undefined.
if (i !== 0) {
node = buildCancelCheckWrapper(node);
}

// Replace the node in the stack with our wrapped one
patchedStack[i] = node;
}

// Put all our expressions in a sequence expression (also called a
// group operator). These will all be executed in order and the value
// of the final expression will be returned.
const sequenceNode = types.builders.sequenceExpression(patchedStack);

path.replace(sequenceNode);
},
});

// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
// Extended functions
visit(ast, {
visitCallExpression(path) {
this.traverse(path);
Expand Down Expand Up @@ -377,7 +428,6 @@ export const extendTransform = (expression: string): { code: string } | undefine
},
});

// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-argument
return print(ast);
} catch (e) {
return;
Expand Down Expand Up @@ -515,3 +565,61 @@ export function extendOptional(
return foundFunction.function(input, args);
};
}

const EXTENDED_SYNTAX_CACHE: Record<string, string> = {};

export function extendSyntax(bracketedExpression: string, forceExtend = false): string {
const chunks = splitExpression(bracketedExpression);

const codeChunks = chunks
.filter((c) => c.type === 'code')
.map((c) => c.text.replace(/("|').*?("|')/, '').trim());

if (
(!codeChunks.some(hasExpressionExtension) || hasNativeMethod(bracketedExpression)) &&
!forceExtend
) {
return bracketedExpression;
}

// If we've seen this expression before grab it from the cache
if (bracketedExpression in EXTENDED_SYNTAX_CACHE) {
return EXTENDED_SYNTAX_CACHE[bracketedExpression];
}

const extendedChunks = chunks.map((chunk): ExpressionChunk => {
if (chunk.type === 'code') {
let output = extendTransform(chunk.text);

// esprima fails to parse bare objects (e.g. `{ data: something }`), we can
// work around this by wrapping it in an parentheses
if (!output?.code && chunk.text.trim()[0] === '{') {
output = extendTransform(`(${chunk.text})`);
}

if (!output?.code) {
throw new ExpressionExtensionError('invalid syntax');
}

let text = output.code;

// We need to cut off any trailing semicolons. These cause issues
// with certain types of expression and cause the whole expression
// to fail.
if (text.trim().endsWith(';')) {
text = text.trim().slice(0, -1);
}

return {
...chunk,
text,
} as ExpressionCode;
}
return chunk;
});

const expression = joinExpression(extendedChunks);
// Cache the expression so we don't have to do this transform again
EXTENDED_SYNTAX_CACHE[bracketedExpression] = expression;
return expression;
}
Loading

0 comments on commit 5251c60

Please sign in to comment.