From e4d98eba6baa129b7414c50956e911c11dff9231 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 19 Jul 2019 13:14:32 +0100 Subject: [PATCH 1/3] use std/esm to make chef compatible with cjs projects. Remove webpack work for node --- .travis.yml | 3 +- Gruntfile.js | 45 +--------------------------- package-lock.json | 5 ++++ package.json | 7 ++--- src/node/cjs.js | 13 ++++++++ src/node/{repl-index.mjs => repl.js} | 8 ++--- 6 files changed, 27 insertions(+), 54 deletions(-) create mode 100644 src/node/cjs.js rename src/node/{repl-index.mjs => repl.js} (86%) diff --git a/.travis.yml b/.travis.yml index 058ba004a1..c1929a9d8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ script: - grunt lint - grunt test - grunt docs - - npm run node-prod - grunt prod --msg="$COMPILE_MSG" - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui before_deploy: @@ -34,7 +33,7 @@ deploy: file_glob: true file: - build/prod/*.zip - - build/node/CyberChef.js + - src/node/cjs.js on: repo: gchq/CyberChef tags: true diff --git a/Gruntfile.js b/Gruntfile.js index ab98608508..df68bee497 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -3,7 +3,6 @@ const webpack = require("webpack"); const HtmlWebpackPlugin = require("html-webpack-plugin"); const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin; -const NodeExternals = require("webpack-node-externals"); const glob = require("glob"); const path = require("path"); @@ -15,7 +14,6 @@ const path = require("path"); * @license Apache-2.0 */ -const NODE_PROD = process.env.NODE_ENV === "production"; module.exports = function (grunt) { grunt.file.defaultEncoding = "utf8"; @@ -36,8 +34,7 @@ module.exports = function (grunt) { grunt.registerTask("node", "Compiles CyberChef into a single NodeJS module.", [ - "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", - "exec:generateNodeIndex", "webpack:node", "webpack:nodeRepl", "chmod:build" + "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex" ]); grunt.registerTask("test", @@ -201,46 +198,6 @@ module.exports = function (grunt) { ] }; }, - node: { - mode: NODE_PROD ? "production" : "development", - target: "node", - entry: "./src/node/index.mjs", - externals: [NodeExternals({ - whitelist: ["crypto-api/src/crypto-api"] - })], - output: { - filename: "CyberChef.js", - path: __dirname + "/build/node", - library: "CyberChef", - libraryTarget: "commonjs2" - }, - plugins: [ - new webpack.DefinePlugin(BUILD_CONSTANTS), - new webpack.optimize.LimitChunkCountPlugin({ - maxChunks: 1 - }) - ], - }, - nodeRepl: { - mode: NODE_PROD ? "production" : "development", - target: "node", - entry: "./src/node/repl-index.mjs", - externals: [NodeExternals({ - whitelist: ["crypto-api/src/crypto-api"] - })], - output: { - filename: "CyberChef-repl.js", - path: __dirname + "/build/node", - library: "CyberChef", - libraryTarget: "commonjs2" - }, - plugins: [ - new webpack.DefinePlugin(BUILD_CONSTANTS), - new webpack.optimize.LimitChunkCountPlugin({ - maxChunks: 1 - }) - ], - } }, "webpack-dev-server": { options: { diff --git a/package-lock.json b/package-lock.json index dfc402b94a..ca59a9cf0e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5097,6 +5097,11 @@ "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==", "dev": true }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, "esmangle": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/esmangle/-/esmangle-1.0.1.tgz", diff --git a/package.json b/package.json index 5dea14a912..bb5d583320 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "type": "git", "url": "https://github.com/gchq/CyberChef/" }, - "main": "build/node/CyberChef.js", + "main": "src/node/cjs.js", "module": "src/node/index.mjs", "bugs": "https://github.com/gchq/CyberChef/issues", "browserslist": [ @@ -108,6 +108,7 @@ "diff": "^4.0.1", "es6-promisify": "^6.0.1", "escodegen": "^1.11.1", + "esm": "^3.2.25", "esmangle": "^1.0.1", "esprima": "^4.0.1", "exif-parser": "^0.1.12", @@ -156,9 +157,7 @@ "scripts": { "start": "grunt dev", "build": "grunt prod", - "node": "NODE_ENV=development grunt node", - "node-prod": "NODE_ENV=production grunt node", - "repl": "grunt node && node build/node/CyberChef-repl.js", + "repl": "node src/node/repl.js", "test": "grunt test", "test-node": "grunt test-node", "testui": "grunt testui", diff --git a/src/node/cjs.js b/src/node/cjs.js new file mode 100644 index 0000000000..a17bce53e3 --- /dev/null +++ b/src/node/cjs.js @@ -0,0 +1,13 @@ +/** + * Export the main ESM module as CommonJS + * + * + * @author d98762656 [d98762625@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +/*eslint no-global-assign: ["off"] */ +require = require("esm")(module); +module.exports = require("./index.mjs"); +module.exports.File = require("./File.mjs"); diff --git a/src/node/repl-index.mjs b/src/node/repl.js similarity index 86% rename from src/node/repl-index.mjs rename to src/node/repl.js index a974e36471..c9ed76f16e 100644 --- a/src/node/repl-index.mjs +++ b/src/node/repl.js @@ -7,9 +7,9 @@ * @license Apache-2.0 */ -import chef from "./index.mjs"; -import repl from "repl"; -import File from "./File.mjs"; +const chef = require("./cjs.js"); +const repl = require("repl"); + /*eslint no-console: ["off"] */ @@ -26,7 +26,7 @@ const replServer = repl.start({ prompt: "chef > ", }); -global.File = File; +global.File = chef.File; Object.keys(chef).forEach((key) => { if (key !== "operations") { From 8548d39318511fd50bdf21234c51329610475ca4 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 2 Aug 2019 11:10:15 +0100 Subject: [PATCH 2/3] add node consumer tests to travis --- .travis.yml | 1 + Gruntfile.js | 46 +++++++++- package-lock.json | 84 +++++++++---------- package.json | 4 +- tests/node/consumers/cjs-consumer.js | 29 +++++++ tests/node/consumers/esm-consumer.mjs | 28 +++++++ .../consumers/esm-deep-import-consumer.mjs | 28 +++++++ 7 files changed, 174 insertions(+), 46 deletions(-) create mode 100644 tests/node/consumers/cjs-consumer.js create mode 100644 tests/node/consumers/esm-consumer.mjs create mode 100644 tests/node/consumers/esm-deep-import-consumer.mjs diff --git a/.travis.yml b/.travis.yml index c1929a9d8a..e99e3903a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ script: - grunt docs - grunt prod --msg="$COMPILE_MSG" - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui + - grunt testnodeconsumer before_deploy: - grunt exec:sitemap - grunt copy:ghPages diff --git a/Gruntfile.js b/Gruntfile.js index df68bee497..e228d785d1 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -48,6 +48,10 @@ module.exports = function (grunt) { "A task which runs all the UI tests in the tests directory. The prod task must already have been run.", ["connect:prod", "exec:browserTests"]); + grunt.registerTask("testnodeconsumer", + "A task which checks whether consuming CJS and ESM apps work with the CyberChef build", + ["exec:setupNodeConsumers", "exec:testCJSNodeConsumer", "exec:testESMNodeConsumer", "exec:testESMDeepImportNodeConsumer", "exec:teardownNodeConsumers"]); + grunt.registerTask("docs", "Compiles documentation in the /docs directory.", ["clean:docs", "jsdoc", "chmod:docs"]); @@ -87,7 +91,8 @@ module.exports = function (grunt) { COMPILE_MSG: JSON.stringify(grunt.option("compile-msg") || grunt.option("msg") || ""), PKG_VERSION: JSON.stringify(pkg.version), }, - moduleEntryPoints = listEntryModules(); + moduleEntryPoints = listEntryModules(), + nodeConsumerTestPath = "~/tmp-cyberchef"; /** @@ -385,7 +390,44 @@ module.exports = function (grunt) { }, nodeTests: { command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs" - } + }, + setupNodeConsumers: { + command: [ + "echo '\n--- Testing node conumers ---'", + "npm link", + `mkdir ${nodeConsumerTestPath}`, + `cp tests/node/consumers/* ${nodeConsumerTestPath}`, + `cd ${nodeConsumerTestPath}`, + "npm link cyberchef" + ].join(";"), + }, + teardownNodeConsumers: { + command: [ + `rm -rf ${nodeConsumerTestPath}`, + "echo '\n--- Node consumer tests complete ---'" + ].join(";"), + }, + testCJSNodeConsumer: { + command: [ + `cd ${nodeConsumerTestPath}`, + "node --no-warnings cjs-consumer.js", + ].join(";"), + stdout: false, + }, + testESMNodeConsumer: { + command: [ + `cd ${nodeConsumerTestPath}`, + "node --no-warnings --experimental-modules esm-consumer.mjs", + ].join(";"), + stdout: false, + }, + testESMDeepImportNodeConsumer: { + command: [ + `cd ${nodeConsumerTestPath}`, + "node --no-warnings --experimental-modules esm-deep-import-consumer.mjs", + ].join(";"), + stdout: false, + }, }, }); }; diff --git a/package-lock.json b/package-lock.json index ca59a9cf0e..8dc3cf36b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3146,15 +3146,6 @@ "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", "dev": true }, - "catharsis": { - "version": "0.8.10", - "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.10.tgz", - "integrity": "sha512-l2OUaz/3PU3MZylspVFJvwHCVfWyvcduPq4lv3AzZ2pJzZCo7kNKFNyatwujD7XgvGkNAE/Jhhbh2uARNwNkfw==", - "dev": true, - "requires": { - "lodash": "^4.17.11" - } - }, "chai-nightwatch": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/chai-nightwatch/-/chai-nightwatch-0.3.0.tgz", @@ -8529,27 +8520,36 @@ } }, "jsdoc": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.2.tgz", - "integrity": "sha512-S2vzg99C5+gb7FWlrK4TVdyzVPGGkdvpDkCEJH1JABi2PKzPeLu5/zZffcJUifgWUJqXWl41Hoc+MmuM2GukIg==", + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.3.tgz", + "integrity": "sha512-Yf1ZKA3r9nvtMWHO1kEuMZTlHOF8uoQ0vyo5eH7SQy5YeIiHM+B0DgKnn+X6y6KDYZcF7G2SPkKF+JORCXWE/A==", "dev": true, "requires": { "@babel/parser": "^7.4.4", "bluebird": "^3.5.4", - "catharsis": "^0.8.10", + "catharsis": "^0.8.11", "escape-string-regexp": "^2.0.0", "js2xmlparser": "^4.0.0", "klaw": "^3.0.0", "markdown-it": "^8.4.2", "markdown-it-anchor": "^5.0.2", - "marked": "^0.6.2", + "marked": "^0.7.0", "mkdirp": "^0.5.1", - "requizzle": "^0.2.2", + "requizzle": "^0.2.3", "strip-json-comments": "^3.0.1", "taffydb": "2.6.2", "underscore": "~1.9.1" }, "dependencies": { + "catharsis": { + "version": "0.8.11", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.11.tgz", + "integrity": "sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g==", + "dev": true, + "requires": { + "lodash": "^4.17.14" + } + }, "escape-string-regexp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", @@ -8565,6 +8565,21 @@ "graceful-fs": "^4.1.9" } }, + "marked": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-0.7.0.tgz", + "integrity": "sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==", + "dev": true + }, + "requizzle": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.3.tgz", + "integrity": "sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ==", + "dev": true, + "requires": { + "lodash": "^4.17.14" + } + }, "strip-json-comments": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", @@ -8994,9 +9009,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "lodash._arraycopy": { "version": "3.0.0", @@ -9088,9 +9103,9 @@ "dev": true }, "lodash.defaultsdeep": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.0.tgz", - "integrity": "sha1-vsECT4WxvZbL6kBbI8FK1kQ6b4E=", + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz", + "integrity": "sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==", "dev": true }, "lodash.escaperegexp": { @@ -9159,15 +9174,15 @@ } }, "lodash.merge": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", - "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, "lodash.mergewith": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz", - "integrity": "sha512-eWw5r+PYICtEBgrBE5hhlT6aAa75f411bgDz/ZL2KZqYV03USvucsxcHUIlGTDTECs1eunpI7HOV7U+WLDvNdQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz", + "integrity": "sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==", "dev": true }, "lodash.once": { @@ -9309,12 +9324,6 @@ "integrity": "sha512-n8zCGjxA3T+Mx1pG8HEgbJbkB8JFUuRkeTZQuIM8iPY6oQ8sWOPRZJDFC9a/pNg2QkHEjjGkhBEl/RSyzaDZ3A==", "dev": true }, - "marked": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.6.2.tgz", - "integrity": "sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==", - "dev": true - }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -11976,15 +11985,6 @@ "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", "dev": true }, - "requizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.2.tgz", - "integrity": "sha512-oJ6y7JcUJkblRGhMByGNcszeLgU0qDxNKFCiUZR1XyzHyVsev+Mxb1tyygxLd1ORsKee1SA5BInFdUwY64GE/A==", - "dev": true, - "requires": { - "lodash": "^4.17.11" - } - }, "resolve": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz", diff --git a/package.json b/package.json index bb5d583320..0c964b6c04 100644 --- a/package.json +++ b/package.json @@ -127,7 +127,7 @@ "kbpgp": "2.1.2", "libbzip2-wasm": "0.0.4", "libyara-wasm": "0.0.12", - "lodash": "^4.17.11", + "lodash": "^4.17.15", "loglevel": "^1.6.3", "loglevel-message-prefix": "^3.0.0", "moment": "^2.24.0", @@ -159,7 +159,7 @@ "build": "grunt prod", "repl": "node src/node/repl.js", "test": "grunt test", - "test-node": "grunt test-node", + "test-node-consumer": "grunt testnodeconsumer", "testui": "grunt testui", "docs": "grunt docs", "lint": "grunt lint", diff --git a/tests/node/consumers/cjs-consumer.js b/tests/node/consumers/cjs-consumer.js new file mode 100644 index 0000000000..1623231201 --- /dev/null +++ b/tests/node/consumers/cjs-consumer.js @@ -0,0 +1,29 @@ +/** + * Tests to ensure that a consuming app can use CJS require + * + * @author d98762625 [d98762625@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +const chef = require("cyberchef"); +const assert = require("assert"); + +const d = chef.bake("Testing, 1 2 3", [ + chef.toHex, + chef.reverse, + { + op: chef.unique, + args: { + delimiter: "Space", + } + }, + { + op: chef.multiply, + args: { + delimiter: "Space", + } + } +]); + +assert.equal(d.value, "630957449041920"); diff --git a/tests/node/consumers/esm-consumer.mjs b/tests/node/consumers/esm-consumer.mjs new file mode 100644 index 0000000000..3536ef0055 --- /dev/null +++ b/tests/node/consumers/esm-consumer.mjs @@ -0,0 +1,28 @@ +/** + * Tests to ensure that a consuming app can use ESM imports + * + * @author d98762625 [d98762625@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import assert from "assert"; +import chef from "cyberchef"; + +const d = chef.bake("Testing, 1 2 3", [ + chef.toHex, + chef.reverse, + { + op: chef.unique, + args: { + delimiter: "Space", + } + }, + { + op: chef.multiply, + args: { + delimiter: "Space", + } + } +]); + +assert.equal(d.value, "630957449041920"); diff --git a/tests/node/consumers/esm-deep-import-consumer.mjs b/tests/node/consumers/esm-deep-import-consumer.mjs new file mode 100644 index 0000000000..58fd921a86 --- /dev/null +++ b/tests/node/consumers/esm-deep-import-consumer.mjs @@ -0,0 +1,28 @@ +/** + * Tests to ensure that a consuming app can use named imports from deep import patch + * + * @author d98762625 [d98762625@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import assert from "assert"; +import { bake, toHex, reverse, unique, multiply } from "cyberchef/src/node/index.mjs"; + +const d = bake("Testing, 1 2 3", [ + toHex, + reverse, + { + op: unique, + args: { + delimiter: "Space", + } + }, + { + op: multiply, + args: { + delimiter: "Space", + } + } +]); + +assert.equal(d.value, "630957449041920"); From 780eecf35bd7c2c7b354f0a4e05403116d19d8d8 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 2 Aug 2019 14:36:40 +0100 Subject: [PATCH 3/3] update readme with Node.js compatibility --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4029fc232f..299b5d8701 100755 --- a/README.md +++ b/README.md @@ -81,6 +81,10 @@ CyberChef is built to support - Mozilla Firefox 35+ - Microsoft Edge 14+ +## Node.js support + +CyberChef is built to fully support Node.js `v10` and partially supports `v12`. Named imports using a deep import specifier does not work in `v12`. For more information, see the Node API page in the project [wiki pages](https://github.com/gchq/CyberChef/wiki) + ## Contributing