diff --git a/.gitignore b/.gitignore index 4747accc..5f2fb9a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /out node_modules +/bundles +/registry/**/.* diff --git a/registry/DependencyResolverPlugin.js b/registry/DependencyResolverPlugin.js new file mode 100644 index 00000000..ee2f32b1 --- /dev/null +++ b/registry/DependencyResolverPlugin.js @@ -0,0 +1,22 @@ +module.exports = function() { + this.plugin('resolve', (request, callback) => { + if (/node_modules/.test(request.path)) { + // Ignore any requests inside node_modules + return callback(); + } + if (/^(?!\.{0,2}\/).+:/.test(request.request)) { + const [parentModule, childModule] = request.request.split(':'); + const obj = Object.assign({}, request, { + request: parentModule, + }); + this.doResolve('resolve', obj, null, (error, result) => { + const obj = Object.assign({}, result, { + request: childModule, + }); + this.doResolve('resolve', obj, null, callback); + }); + } else { + callback(); + } + }); +} diff --git a/registry/build-worker.js b/registry/build-worker.js new file mode 100644 index 00000000..e542f06c --- /dev/null +++ b/registry/build-worker.js @@ -0,0 +1,41 @@ +const Queue = require('bee-queue'); +const child_process = require('child_process'); +const logger = require('./logger'); +const path = require('path'); + +const BUNDLE_DIR = path.resolve(process.env.BUNDLE_DIR); + +const buildQueue = new Queue('build'); + +buildQueue.process(2, (job, done) => { + const {tool, version} = job.data; + const key = `${tool}@${version}`; + const diff = measure(); + + child_process.fork( + path.join(__dirname, 'build.js'), + [tool, version], + { + env: { + BUNDLE_DIR, + PATH: process.env.PATH, + }, + } + ) + .on('close', code => { + if (code) { + const error = new Error( + `Unable to build package ${tool}@${version}` + ); + logger.error(error); + return done(error); + } + logger.log(`Built ${tool}@${version} in ${diff().toFixed(2)} sec`); + done(null, key); + }); +}); + +function measure() { + const startTime = Date.now(); + return () => ((Date.now() - startTime) / 1000); +} diff --git a/registry/build.js b/registry/build.js new file mode 100755 index 00000000..69e192c9 --- /dev/null +++ b/registry/build.js @@ -0,0 +1,215 @@ +#!/usr/bin/env node +/* eslint no-console: 0 */ + +require('isomorphic-fetch'); +const fs = require('fs-promise'); +const path = require('path'); +const semver = require('semver'); +const child_process = require('child_process'); +const webpack = require('webpack'); + +const REGISTRY_DIR = path.join(__dirname, 'tools'); +const BUNDLE_DIR = process.env.BUNDLE_DIR; + +const toolID = process.argv[2]; +const version = process.argv[3] || 'latest'; + +if (!(toolID && version)) { + console.error('You need to provide a tool ID and a version to build'); +} + +const diff = measure(); +buildVersion(toolID, version) + .then(() => console.log(`Built in ${diff().toFixed(2)} sec.`)) + .catch(error => { + console.log(error); + process.exit(1); + }); + +function buildVersion(toolID, version) { + const toolDir = path.join(REGISTRY_DIR, toolID); + if (!exists(toolDir)) { + return Promise.reject(new Error(`${toolID} doesn't exist.`)); + } + const toolConfig = require(path.resolve(path.join(toolDir, 'package.js'))); + const latest = version === 'latest'; + + return fetchInfo(toolID, version).then(npmPkg => { + const version = npmPkg.version; + + const versionConfig = toolConfig.versions.find( + config => semver.satisfies(version, config.dependencies[toolID]) + ); + if (!versionConfig) { + throw new Error(`No suitable package found for ${toolID}@${version}.`); + } + + return install(npmPkg, toolDir, versionConfig) + .then(cacheDir => + bundleVersion( + cacheDir, + versionConfig, + npmPkg, + latest + ).then(() => fs.remove(cacheDir)) + ); + }) +} + +function fetchInfo(name, version) { + console.log(`Fetching ${name}...`); + return fetch(`https://registry.npmjs.org/${name}/${version}`) + .then(response => { + if (!response.ok) { + throw new Error(response.status); + } + return response.json(); + }); +} + +function matchesVersion(toolID, versionPath, version) { + return fs.readJSON(versionPath) + .then( + pkg => semver.satisfies(version, pkg.dependencies[toolID]) ? + versionPath : + null + ); +} + +function preparePackage(npmPkg, packagePath) { + return fs.readJSON(packagePath) + .then(localPkg => { + // Get all versions that need to be built + const acceptedVersion = localPkg.dependencies[npmPkg.name]; + let versions = Object.keys(npmPkg.versions) + .filter(v => semver.satisfies(v, acceptedVersion)) + .filter(x => x); + if (versions.length === 0) { + // nothing to do + console.log(`Nothing to do for ${npmPkg.name}...`); + } + // versions = getLatestMinorVersions(versions); + versions = versions.slice(0,1); + // Install specific version and its dependencies + return Promise.all(versions.map( + v => install(npmPkg.versions[v], packagePath) + .then(bundleVersion) + .then(() => console.log('Bundle built')) + )); + }); +} + +function bundleVersion(packagePath, versionConfig, npmPkg, latest) { + return new Promise((resolve, reject) => { + const name = `${npmPkg.name}@${latest ? 'latest' : npmPkg.version}`; + webpack( + Object.assign( + versionConfig.webpackConfig, + { + entry: path.join(packagePath, versionConfig.main), + context: __dirname, + output: { + filename: `${name}.js`, + path: BUNDLE_DIR, + libraryTarget: 'amd', + library: `${toolID}/${version}`, + }, + } + ), + (err, stats) => { + if (err) { + return reject(err); + } + if (stats.hasErrors()) { + console.error(stats.toJson().errors); + return reject('Bundle compilation error'); + } + resolve(); + } + ); + }); + +} + +function install(npmVersion, toolDir, versionConfig) { + console.log(`Installing ${npmVersion.name}@${npmVersion.version}...`); + // Create version folder + const cacheDir = path.join(toolDir, '..', `.${npmVersion.name}@${npmVersion.version}`); + const packagePath = path.join(cacheDir, 'package.json'); + return fs.ensureDir(cacheDir) + .then(() => Promise.all([ + fs.copy(toolDir, cacheDir), + fs.writeJSON(packagePath, pick(versionConfig, 'main', 'dependencies')), + ])) + .then(() => run( + `yarn add --exact --no-lockfile --no-bin-links --prod --no-progress ${npmVersion.name}@${npmVersion.version}`, + {cwd: cacheDir} + )) + .then(() => run( + 'yarn --prefer-offline --prod --no-lockfile --no-progress', + {cwd: cacheDir} + )) + .then(() => cacheDir); +} + +function pick(obj, ...props) { + const result = {}; + for (var prop in obj) { + result[prop] = obj[prop]; + } + return result; +} + +function getPackagePath(p) { + return path.join(p, 'package.json'); +} + +function exists(p) { + try { + fs.accessSync(p, fs.constants.R_OK); + return true; + } catch(e) {} + return false; +} + +// Given a list of versions, these function removes all but the latest patch +// version. E.g. +// In: [1.0.1, 1.0.2, 1.1.2, 1.1.3, 1.1.4] +// Out: [1.0.2, 1.1.4] +function getLatestMinorVersions(versions) { + const last = versions.length - 1; + return versions.sort(semver.compare) + .filter((v, i) => { + if (i === last) { + return true; + } + if (semver.diff(v, versions[i+1]) === 'minor') { + return true; + } + return false; + }); +} + +function run(command, options) { + options = Object.assign({}, options, {env: {PATH: process.env.PATH}}); + return new Promise((resolve, reject) => { + child_process.exec(command, options, (error, stdout, stderr) => { + if (error) { + reject({error, stdout, stderr}); + } else { + resolve({stdout, stderr}); + } + }); + }); +} + +function withoutHidden(p) { + return p[0] !== '.'; +} + +function measure() { + const start = Date.now(); + return function() { + return (Date.now() - start) / 1000; + } +} diff --git a/registry/categories/js/codeExample.txt b/registry/categories/js/codeExample.txt new file mode 100644 index 00000000..6fd12980 --- /dev/null +++ b/registry/categories/js/codeExample.txt @@ -0,0 +1,19 @@ +/** + * Paste or drop some JavaScript here and explore + * the syntax tree created by chosen parser. + * You can use all the cool new features from ES6 + * and even more. Enjoy! + */ + +let tips = [ + "Click on any AST node with a '+' to expand it", + + "Hovering over a node highlights the \ + corresponding part in the source code", + + "Shift click on an AST node expands the whole substree" +]; + +function printTips() { + tips.forEach((tip, i) => console.log(`Tip ${i}:` + tip)); +} diff --git a/registry/categories/js/index.js b/registry/categories/js/index.js new file mode 100644 index 00000000..0041ee04 --- /dev/null +++ b/registry/categories/js/index.js @@ -0,0 +1,4 @@ +export const id = 'javascript'; +export const displayName = 'JavaScript'; +export const mimeTypes = ['text/javascript']; +export const fileExtension = 'js'; diff --git a/registry/generate-inventory.js b/registry/generate-inventory.js new file mode 100755 index 00000000..206af621 --- /dev/null +++ b/registry/generate-inventory.js @@ -0,0 +1,108 @@ +#!/usr/bin/env node +/* eslint no-console: 0 */ + +require('isomorphic-fetch'); +const fs = require('fs-promise'); +const path = require('path'); +const semver = require('semver'); + +const REGISTRY_DIR = path.join(__dirname, 'tools'); +const BUNDLE_DIR = process.env.BUNDLE_DIR; + +const diff = measure(); +findPackages() + .then(pkgs => Promise.all(pkgs.map(getVersions))) + .then(writeInventoryFile) + .then(() => console.log(`Built in ${diff().toFixed(2)} sec.`)) + .catch(error => console.error(error)); + + +function findPackages() { + return fs.readdir(REGISTRY_DIR) + .then(tools => tools.filter(isDir).filter(withoutHidden)) + .then(tools => tools.map( + tool => require(path.join(REGISTRY_DIR, tool, 'package.js')) + )); +} + +function getVersions(pkg) { + return fetchInfo(pkg.name).then(npmPkg => { + const acceptedVersions = pkg.versions.map(v => v.dependencies[pkg.name]); + const versions = Object.keys(npmPkg.versions) + .filter(v => acceptedVersions.some(range => semver.satisfies(v, range))); + + pkg.availableVersions = semver.lt(npmPkg['dist-tags'].latest, '1.0.0') ? + getLatestPatchVersions(versions) : + getLatestMinorVersions(versions); + return pkg; + }); +} + +function writeInventoryFile(data) { + return fs.writeJSON(path.join(BUNDLE_DIR, 'inventory.json'), data.map(pkg => ({ + name: pkg.name, + displayName: pkg.displayName, + homepage: pkg.homepage, + versions: pkg.availableVersions, + category: pkg.category, + }))); +} + + +function fetchInfo(name) { + console.log(`Fetching ${name}...`); + return fetch(`https://registry.npmjs.org/${name}`) + .then(response => { + if (!response.ok) { + throw new Error(response.status); + } + return response.json(); + }); +} + +// Given a list of versions, these function removes all but the latest patch +// version. E.g. +// In: [1.0.1, 1.0.2, 1.1.2, 1.1.3, 1.1.4] +// Out: [1.0.2, 1.1.4] +function getLatestMinorVersions(versions) { + const last = versions.length - 1; + return versions.sort(semver.compare) + .filter((v, i) => { + if (i === last) { + return true; + } + if (semver.diff(v, versions[i+1]) === 'minor') { + return true; + } + return false; + }); +} + +function getLatestPatchVersions(versions) { + const last = versions.length - 1; + return versions.sort(semver.compare) + .filter((v, i) => { + if (i === last) { + return true; + } + if (semver.diff(v, versions[i+1]) === 'patch') { + return true; + } + return false; + }); +} + +function withoutHidden(p) { + return p[0] !== '.'; +} + +function isDir(p) { + return !/\..{2,}$/.test(p); +} + +function measure() { + const start = Date.now(); + return function() { + return (Date.now() - start) / 1000; + } +} diff --git a/registry/logger.js b/registry/logger.js new file mode 100644 index 00000000..8dd34fc9 --- /dev/null +++ b/registry/logger.js @@ -0,0 +1,19 @@ +const winston = require('winston'); +winston.level = 'info'; + +exports.log = msg => { + winston.log('info', `${getDateString()} ${msg}`); +}; + +exports.error = msg => { + winston.log('error', `${getDateString()} ${msg}`); +}; + +function getDateString() { + const now = new Date(); + return `${now.getYear() + 1900}-${pad(now.getUTCMonth()+1)}-${pad(now.getUTCDate())} ${pad(now.getUTCHours())}:${pad(now.getUTCMinutes())}:${pad(now.getUTCSeconds())}`; +} + +function pad(v) { + return v < 10 ? '0' + v : v; +} diff --git a/registry/package.json b/registry/package.json new file mode 100644 index 00000000..2fc83c9e --- /dev/null +++ b/registry/package.json @@ -0,0 +1,29 @@ +{ + "dependencies": { + "babel-core": "^6.22.1", + "babel-loader": "^6.2.10", + "babel-plugin-transform-object-rest-spread": "^6.22.0", + "babel-plugin-transform-regenerator": "^6.22.0", + "babel-plugin-transform-runtime": "^6.22.0", + "babel-preset-es2015": "^6.22.0", + "babel-preset-react": "^6.22.0", + "babel-preset-stage-0": "^6.22.0", + "bee-queue": "^0.3.0", + "eslint": "^3.11.1", + "eslint-import-resolver-webpack": "^0.8.0", + "eslint-plugin-import": "^2.2.0", + "eslint-plugin-react": "^6.8.0", + "fs-promise": "^1.0.0", + "imports-loader": "^0.7.0", + "isomorphic-fetch": "^2.2.1", + "raw-loader": "^0.5.1", + "semver": "^5.3.0", + "webpack": "^2.2.1", + "winston": "^2.3.1", + "wrapper-webpack-plugin": "^0.1.11" + }, + "scripts": { + "start-worker": "BUNDLE_DIR=../bundles node build-worker.js", + "lint": "eslint ./" + } +} diff --git a/registry/tools/acorn/base.js b/registry/tools/acorn/base.js new file mode 100644 index 00000000..c17ff695 --- /dev/null +++ b/registry/tools/acorn/base.js @@ -0,0 +1,68 @@ +import React from 'react'; +import defaultParserInterface from '../../utils/defaultESTreeParserInterface.js'; +import SettingsRenderer from '../../utils/SettingsRenderer'; +import * as category from '../../categories/js'; +import codeExample from '../../categories/js/codeExample.txt'; + +import pkg from 'acorn/package.json'; + +const ID = 'acorn'; + +export default { + ...defaultParserInterface, + + displayName: ID, + homepage: pkg.homepage, + category, + codeExample, + + locationProps: new Set(['range', 'loc', 'start', 'end']), + + // load needs to be implement in the "child" objects + // and pass `parse` and `loose` to the callback + // + // load() { + // return Promise.resolve({ + // parse: jsxInject(acorn).parse, + // loose: parse_dammit, + // }); + // }, + + parse(parser, code, options={}) { + options = Object.assign({}, this.defaultOptions, options); + const parse = options.loose ? + parser.loose : + parser.parse; + + // put deep option into correspondent place + return parse(code, { + ...options, + plugins: options['plugins.jsx'] && !options.loose ? { jsx: true } : {}, + }); + }, + + nodeToRange(node) { + if (typeof node.start === 'number') { + return [node.start, node.end]; + } + }, + + renderSettings(parserSettings, onChange) { + return ( +