Merge branch 'feature/resource-mapping-component' into NODE-398-resou…

…rce-mapper-ui-P2

* feature/resource-mapping-component: (31 commits)
  feat(Date & Time Node): Overhaul of the node (#5904)
  refactor: Add deprecation notice for WEBHOOK_TUNNEL_URL (#6194)
  refactor: Add deprecation notice for WEBHOOK_TUNNEL_URL (#6194)
  feat(HubSpot Node): Overhaul the HubSpot Node (#4337)
  ci: Create a nightly v1 docker image (no-changelog) (#6197)
  refactor(core): Add deprecation notice for own mode (#6195)
  feat(core): Update config defaults for for v1 (no-changelog) (#6196)
  test(Read Binary Files Node): Unit tests (no-changelog) (#5459)
  refactor: Add deprecation notice for MySQL and MariaDB (#6189)
  test: Create custom jest error messages using jest-expect-message (no-changelog) (#5666)
  fix(core): Move nodeExecute InternalHook calls to hookFunctionsSave (#6193)
  fix(FTP Node): Use filename instead of remote filepath for downloaded binary data (#6170)
  ci: Fix test database cleanup (no-changelog) (#6188)
  refactor(core): Delete boilerplate code across migrations (no-changelog) (#5254)
  refactor(editor): Add infix to Pinia stores (no-changelog) (#6149)
  ci: Fix linting issues on master (no-changelog) (#6186)
  fix(editor): Update and add design system checkbox component to Editor (#6178)
  fix(editor): Display SSO entry in Settings on Cloud (#6181)
  feat(Code Node): Add Python support (#4295)
  fix(editor): Update and fix storybook (was failing to run in local dev mode) (#6180)
  ...

# Conflicts:
#	packages/editor-ui/src/stores/nodeTypes.store.ts

.github/workflows/docker-image-nightly-v1.yml

@@ -0,0 +1,42 @@
+name: Docker Nightly Image - V1
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: pnpm/action-setup@v2.2.4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+      - run: npm install --prefix=.github/scripts --no-package-lock
+
+      - name: Bump package versions to 1.0.0
+        run: |
+          RELEASE_TYPE=major node .github/scripts/bump-versions.mjs
+          pnpm i --lockfile-only
+
+      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./docker/images/n8n-custom/Dockerfile
+          platforms: linux/amd64
+          provenance: false
+          push: true
+          tags: ${{ secrets.DOCKER_USERNAME }}/n8n:1.0.0-rc
+          no-cache: true

jest.config.js

@@ -22,6 +22,7 @@ const config = {
 	moduleNameMapper: {
 		'^@/(.*)$': '<rootDir>/src/$1',
 	},
+	setupFilesAfterEnv: ['jest-expect-message'],
 	collectCoverage: true,
 	coverageReporters: [process.env.COVERAGE_REPORT === 'true' ? 'text' : 'text-summary'],
 	collectCoverageFrom: ['src/**/*.ts'],

package.json

@@ -46,6 +46,7 @@
     "cypress-real-events": "^1.7.6",
     "jest": "^29.5.0",
     "jest-environment-jsdom": "^29.5.0",
+    "jest-expect-message": "^1.1.3",
     "jest-mock": "^29.5.0",
     "jest-mock-extended": "^3.0.4",
     "nock": "^13.2.9",
@@ -79,6 +80,7 @@
       "tslib": "^2.5.0",
       "ts-node": "^10.9.1",
       "typescript": "^5.0.3",
+      "xlsx": "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz",
       "xml2js": "^0.5.0",
       "cpy@8>globby": "^11.1.0",
       "qqjs>globby": "^11.1.0"

packages/cli/.eslintrc.js

@@ -11,7 +11,6 @@ module.exports = {
 	ignorePatterns: [
 		'jest.config.js',
 		// TODO: Remove these
-		'src/databases/migrations/**',
 		'src/databases/ormconfig.ts',
 	],
 

packages/cli/src/Db.ts

@@ -7,7 +7,7 @@ import { Container } from 'typedi';
 import type { DataSourceOptions as ConnectionOptions, EntityManager, LoggerOptions } from 'typeorm';
 import { DataSource as Connection } from 'typeorm';
 import type { TlsOptions } from 'tls';
-import type { DatabaseType, IDatabaseCollections } from '@/Interfaces';
+import type { IDatabaseCollections } from '@/Interfaces';
 
 import config from '@/config';
 
@@ -19,6 +19,8 @@ import {
 	getPostgresConnectionOptions,
 	getSqliteConnectionOptions,
 } from '@db/config';
+import { wrapMigration } from '@db/utils/migrationHelpers';
+import type { DatabaseType, Migration } from '@db/types';
 import {
 	AuthIdentityRepository,
 	AuthProviderSyncHistoryRepository,
@@ -119,7 +121,7 @@ export async function init(
 		synchronize: false,
 		logging: loggingOption,
 		maxQueryExecutionTime,
-		migrationsTransactionMode: 'each',
+		migrationsRun: false,
 	});
 
 	connection = new Connection(connectionOptions);
@@ -136,15 +138,17 @@ export async function init(
 		await connection.query(`SET search_path TO ${searchPath.join(',')};`);
 	}
 
+	(connectionOptions.migrations as Migration[]).forEach(wrapMigration);
+
 	if (!testConnectionOptions && dbType === 'sqlite') {
 		// This specific migration changes database metadata.
 		// A field is now nullable. We need to reconnect so that
 		// n8n knows it has changed. Happens only on sqlite.
 		let migrations = [];
 		try {
-			const entityPrefix = config.getEnv('database.tablePrefix');
+			const tablePrefix = config.getEnv('database.tablePrefix');
 			migrations = await connection.query(
-				`SELECT id FROM ${entityPrefix}migrations where name = "MakeStoppedAtNullable1607431743769"`,
+				`SELECT id FROM ${tablePrefix}migrations where name = "MakeStoppedAtNullable1607431743769"`,
 			);
 		} catch (error) {
 			// Migration table does not exist yet - it will be created after migrations run for the first time.

packages/cli/src/Interfaces.ts

@@ -32,6 +32,7 @@ import type { FindOperator } from 'typeorm';
 
 import type { ChildProcess } from 'child_process';
 
+import type { DatabaseType } from '@db/types';
 import type { AuthProviderType } from '@db/entities/AuthIdentity';
 import type { Role } from '@db/entities/Role';
 import type { SharedCredentials } from '@db/entities/SharedCredentials';
@@ -160,7 +161,6 @@ export type ICredentialsDecryptedDb = ICredentialsBase & ICredentialsDecrypted;
 
 export type ICredentialsDecryptedResponse = ICredentialsDecryptedDb;
 
-export type DatabaseType = 'mariadb' | 'postgresdb' | 'mysqldb' | 'sqlite';
 export type SaveExecutionDataType = 'all' | 'none';
 
 export interface IExecutionBase {

packages/cli/src/WorkflowExecuteAdditionalData.ts

@@ -515,8 +515,24 @@ export function hookFunctionsPreExecute(parentProcessMode?: string): IWorkflowEx
  */
 function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
 	return {
-		nodeExecuteBefore: [],
-		nodeExecuteAfter: [],
+		nodeExecuteBefore: [
+			async function (this: WorkflowHooks, nodeName: string): Promise<void> {
+				void Container.get(InternalHooks).onNodeBeforeExecute(
+					this.executionId,
+					this.workflowData,
+					nodeName,
+				);
+			},
+		],
+		nodeExecuteAfter: [
+			async function (this: WorkflowHooks, nodeName: string): Promise<void> {
+				void Container.get(InternalHooks).onNodePostExecute(
+					this.executionId,
+					this.workflowData,
+					nodeName,
+				);
+			},
+		],
 		workflowExecuteBefore: [],
 		workflowExecuteAfter: [
 			async function (
@@ -1261,26 +1277,6 @@ export function getWorkflowHooksWorkerMain(
 	hookFunctions.nodeExecuteBefore = [];
 	hookFunctions.nodeExecuteAfter = [];
 
-	hookFunctions.nodeExecuteBefore.push(async function (
-		this: WorkflowHooks,
-		nodeName: string,
-	): Promise<void> {
-		void Container.get(InternalHooks).onNodeBeforeExecute(
-			this.executionId,
-			this.workflowData,
-			nodeName,
-		);
-	});
-	hookFunctions.nodeExecuteAfter.push(async function (
-		this: WorkflowHooks,
-		nodeName: string,
-	): Promise<void> {
-		void Container.get(InternalHooks).onNodePostExecute(
-			this.executionId,
-			this.workflowData,
-			nodeName,
-		);
-	});
 	return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters);
 }
 
@@ -1313,27 +1309,7 @@ export function getWorkflowHooksMain(
 	}
 
 	if (!hookFunctions.nodeExecuteBefore) hookFunctions.nodeExecuteBefore = [];
-	hookFunctions.nodeExecuteBefore?.push(async function (
-		this: WorkflowHooks,
-		nodeName: string,
-	): Promise<void> {
-		void Container.get(InternalHooks).onNodeBeforeExecute(
-			this.executionId,
-			this.workflowData,
-			nodeName,
-		);
-	});
 	if (!hookFunctions.nodeExecuteAfter) hookFunctions.nodeExecuteAfter = [];
-	hookFunctions.nodeExecuteAfter.push(async function (
-		this: WorkflowHooks,
-		nodeName: string,
-	): Promise<void> {
-		void Container.get(InternalHooks).onNodePostExecute(
-			this.executionId,
-			this.workflowData,
-			nodeName,
-		);
-	});
 
 	return new WorkflowHooks(hookFunctions, data.executionMode, executionId, data.workflowData, {
 		sessionId: data.sessionId,

packages/cli/src/api/e2e.api.ts

@@ -36,30 +36,39 @@ type Feature = keyof typeof enabledFeatures;
 
 Container.get(License).isFeatureEnabled = (feature: Feature) => enabledFeatures[feature] ?? false;
 
-const tablesNotToTruncate = ['sqlite_sequence'];
+const tablesToTruncate = [
+	'auth_identity',
+	'auth_provider_sync_history',
+	'event_destinations',
+	'shared_workflow',
+	'shared_credentials',
+	'webhook_entity',
+	'workflows_tags',
+	'credentials_entity',
+	'tag_entity',
+	'workflow_statistics',
+	'workflow_entity',
+	'execution_entity',
+	'settings',
+	'installed_packages',
+	'installed_nodes',
+	'user',
+	'role',
+	'variables',
+];
 
 const truncateAll = async () => {
 	const connection = Db.getConnection();
-	const allTables: Array<{ name: string }> = await connection.query(
-		"SELECT name FROM sqlite_master WHERE type='table';",
-	);
-
-	// Disable foreign key constraint checks
-	await connection.query('PRAGMA foreign_keys = OFF;');
 
-	for (const { name: table } of allTables) {
+	for (const table of tablesToTruncate) {
 		try {
-			if (tablesNotToTruncate.includes(table)) continue;
 			await connection.query(
 				`DELETE FROM ${table}; DELETE FROM sqlite_sequence WHERE name=${table};`,
 			);
 		} catch (error) {
 			console.warn('Dropping Table for E2E Reset error: ', error);
 		}
 	}
-
-	// Re-enable foreign key constraint checks
-	await connection.query('PRAGMA foreign_keys = ON;');
 };
 
 const setupUserManagement = async () => {

packages/cli/src/commands/BaseCommand.ts

@@ -55,6 +55,24 @@ export abstract class BaseCommand extends Command {
 			this.exitWithCrash('There was an error initializing DB', error),
 		);
 
+		if (process.env.WEBHOOK_TUNNEL_URL) {
+			LoggerProxy.warn(
+				'You are still using the WEBHOOK_TUNNEL_URL environment variable. It has been deprecated and will be removed in a future version of n8n. Please switch to using WEBHOOK_URL instead.',
+			);
+		}
+		const dbType = config.getEnv('database.type');
+
+		if (['mysqldb', 'mariadb'].includes(dbType)) {
+			LoggerProxy.warn(
+				'Support for MySQL/MariaDB has been deprecated and will be removed with an upcoming version of n8n. Please migrate to PostgreSQL.',
+			);
+		}
+		if (process.env.EXECUTIONS_PROCESS === 'own') {
+			LoggerProxy.warn(
+				'Own mode has been deprecated and will be removed in a future version of n8n. If you need the isolation and performance gains, please consider using queue mode.',
+			);
+		}
+
 		this.instanceId = this.userSettings.instanceId ?? '';
 		await Container.get(PostHogClient).init(this.instanceId);
 		await Container.get(InternalHooks).init(this.instanceId);

packages/cli/src/config/schema.ts

@@ -4,6 +4,7 @@ import path from 'path';
 import convict from 'convict';
 import { UserSettings } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
+import { IS_V1_RELEASE } from '@/constants';
 
 convict.addFormat({
 	name: 'nodes-list',
@@ -129,7 +130,7 @@ export const schema = {
 		},
 		mysqldb: {
 			database: {
-				doc: 'MySQL Database',
+				doc: '[DEPRECATED] MySQL Database',
 				format: String,
 				default: 'n8n',
 				env: 'DB_MYSQLDB_DATABASE',
@@ -228,9 +229,9 @@ export const schema = {
 		// If this option gets set to "main" it will run them in the
 		// main-process instead.
 		process: {
-			doc: 'In what process workflows should be executed',
+			doc: 'In what process workflows should be executed. Note: Own mode has been deprecated and will be removed in a future version as well as this setting.',
 			format: ['main', 'own'] as const,
-			default: 'own',
+			default: IS_V1_RELEASE ? 'main' : 'own',
 			env: 'EXECUTIONS_PROCESS',
 		},
 
@@ -943,7 +944,7 @@ export const schema = {
 	push: {
 		backend: {
 			format: ['sse', 'websocket'] as const,
-			default: 'sse',
+			default: IS_V1_RELEASE ? 'websocket' : 'sse',
 			env: 'N8N_PUSH_BACKEND',
 			doc: 'Backend to use for push notifications',
 		},

packages/cli/src/constants.ts

@@ -3,6 +3,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import { readFileSync } from 'fs';
 import { resolve, join, dirname } from 'path';
+import { major } from 'semver';
 import type { n8n } from 'n8n-core';
 import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES, UserSettings } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
@@ -29,6 +30,7 @@ export function getN8nPackageJson() {
 export const START_NODES = ['n8n-nodes-base.start', 'n8n-nodes-base.manualTrigger'];
 
 export const N8N_VERSION = getN8nPackageJson().version;
+export const IS_V1_RELEASE = major(N8N_VERSION) > 0;
 
 export const NODE_PACKAGE_PREFIX = 'n8n-nodes-';
 

packages/cli/src/databases/config.ts

@@ -8,7 +8,7 @@ import { entities } from './entities';
 import { mysqlMigrations } from './migrations/mysqldb';
 import { postgresMigrations } from './migrations/postgresdb';
 import { sqliteMigrations } from './migrations/sqlite';
-import type { DatabaseType } from '@/Interfaces';
+import type { DatabaseType } from '@db/types';
 import config from '@/config';
 
 const entitiesDir = path.resolve(__dirname, 'entities');
@@ -35,7 +35,6 @@ const getDBConnectionOptions = (dbType: DatabaseType) => {
 	return {
 		entityPrefix,
 		entities: Object.values(entities),
-		migrationsRun: false,
 		migrationsTableName: `${entityPrefix}migrations`,
 		cli: { entitiesDir, migrationsDir },
 		...connectionDetails,