Merge branch 'cleanup'

Various changes to improve the source organisation and build process.
Makes ./extension an output-only folder; cleaned up before every build.

Plenty of intended improvements are not yet addressed; see issue #112.

.gitignore

@@ -1,7 +1,5 @@
 node_modules
+extension
 dist
 .chrome-extension-key.pem
-extension/lib
-extension/**/*.js
-extension/**/*.css
 jest_0/

.travis.yml

@@ -2,7 +2,8 @@ language: node_js
 node_js:
     - "6"
 script:
-    - npm run test # first run tests, to fail quickly.
+    - npm run test
+    - npm run lint
     - npm run build
 cache: yarn
 notifications:

Readme.md

@@ -59,7 +59,7 @@ this personal web based on [React]+[Redux].
 2. Get [Node/NPM] and [yarn] (`npm install -g yarn`).
 3. Run `make` to install dependencies and compile the source files.
 4. Load it in Firefox or Chromium/Chrome:
-    * In Firefox (≥49): run `npm run fx-run` (or run [web-ext] directly for more control).
+    * In Firefox (≥49): run `npm run firefox` (or run [web-ext] directly for more control).
       Alternatively, go to `about:debugging`, choose 'Load Temporary Add-on', and pick
       `extension/manifest.json` from this repo.
     * In Chromium/Chrome: go to Tools→Extensions (`chrome://extensions`), enable 'Developer mode',
@@ -70,7 +70,7 @@ this personal web based on [React]+[Redux].
 If the steps above worked, running `npm run watch` will trigger a quick recompilation every time a
 source file has been modified.
 
-If you are testing in Firefox through `npm run fx-run`/`web-ext`, the extension should also reload
+If you are testing in Firefox through `npm run firefox`/`web-ext`, the extension should also reload
 automatically. Otherwise, manually press the reload button in the extension list.
 
 

gulpfile.babel.js

@@ -1,18 +1,23 @@
 import fs from 'fs'
+import path from 'path'
 import { exec as nodeExec} from 'child_process'
 import pify from 'pify'
+import streamToPromise from 'stream-to-promise'
 import gulp from 'gulp'
-import uglify from 'gulp-uglify'
+import addsrc from 'gulp-add-src'
+import clipEmptyFiles from 'gulp-clip-empty-files'
+import concatCss from 'gulp-concat-css'
 import identity from 'gulp-identity'
 import source from 'vinyl-source-stream'
 import buffer from 'vinyl-buffer'
+import uglify from 'gulp-uglify'
+import eslint from 'gulp-eslint'
+import stylelint from 'gulp-stylelint'
 import browserify from 'browserify'
 import watchify from 'watchify'
 import babelify from 'babelify'
 import envify from 'loose-envify/custom'
-import eslint from 'gulp-eslint'
 import uglifyify from 'uglifyify'
-import path from 'path'
 import cssModulesify from 'css-modulesify'
 import cssnext from 'postcss-cssnext'
 
@@ -22,39 +27,20 @@ const exec = pify(nodeExec)
 // === Tasks for building the source code; result is put into ./extension ===
 
 const staticFiles = {
+    'src/manifest.json': 'extension',
+    'src/*.html': 'extension',
     'node_modules/webextension-polyfill/dist/browser-polyfill.js': 'extension/lib',
     'node_modules/pdfjs-dist/build/pdf.worker.min.js': 'extension/lib',
     'node_modules/semantic-ui-css/semantic.min.css': 'extension/lib/semantic-ui',
     'node_modules/semantic-ui-css/themes/**/*': 'extension/lib/semantic-ui/themes',
 }
 
 const sourceFiles = [
-    {
-        entries: ['./src/background.js'],
-        output: 'background.js',
-        destination: './extension',
-    },
-    {
-        entries: ['./src/content_script.js'],
-        output: 'content_script.js',
-        destination: './extension',
-    },
-    {
-        entries: ['./src/overview/main.jsx'],
-        output: 'overview.js',
-        destination: './extension/overview',
-        cssOutput: 'style.css',
-    },
-    {
-        entries: ['./src/page-viewer/localpage.js'],
-        output: 'localpage.js',
-        destination: './extension/page-viewer',
-    },
-    {
-        entries: ['./src/popup/popup.js'],
-        output: 'popup.js',
-        destination: './extension/popup',
-    },
+    'background.js',
+    'content_script.js',
+    'overview/overview.jsx',
+    'local-page/local-page.js',
+    'popup/popup.js',
 ]
 
 const browserifySettings = {
@@ -63,8 +49,17 @@ const browserifySettings = {
     paths: ['.'],
 }
 
-function createBundle({entries, output, destination, cssOutput},
-                      {watch = false, production = false}) {
+async function createBundle({filePath, watch = false, production = false}) {
+    const { dir, name, ext } = path.parse(filePath)
+    const entries = [path.join('src', filePath)]
+    const destination = path.join('extension', dir)
+    const output = `${name}.js` // ignore original filename extension, to replace jsx with js.
+
+    // Hard-code the inclusion of any css file with the same name as the script.
+    // We append any css-modules imported from the script to this css file.
+    const cssInputPath = path.join('src', dir, `${name}.css`)
+    const cssOutput = `${name}.css`
+
     let b = watch
         ? watchify(browserify({...watchify.args, ...browserifySettings, entries}))
             .on('update', bundle)
@@ -74,21 +69,27 @@ function createBundle({entries, output, destination, cssOutput},
         NODE_ENV: production ? 'production' : 'development',
     }), {global: true})
 
-    if (cssOutput) {
-        b.plugin(cssModulesify, {
-            global: true,
-            output: path.join(destination, cssOutput),
-            postcssBefore: [
-                cssnext,
-            ],
-        })
-    }
-
-    if (production) {
-        b.transform(uglifyify, {global: true})
-    }
+    b.plugin(cssModulesify, {
+        global: true, // for importing css modules from e.g. react-datepicker.
+        rootDir: path.join('src', dir),
+        // output: path.join(destination, cssOutput), // We read the stream instead (see below)
+        postcssBefore: [
+            cssnext,
+        ],
+    })
+    b.on('css stream', stream => {
+        // Append the css-modules output to the script's eponymous plain css file (if any).
+        // TODO resolve & copy @import and url()s
+        stream
+            .pipe(source('css-modules-output.css')) // pretend the streamed data had this filename.
+            .pipe(buffer()) // concatCss & clipEmptyFiles do not support streamed files.
+            .pipe(addsrc.prepend(cssInputPath))
+            .pipe(concatCss(cssOutput, {inlineImports: false}))
+            .pipe(clipEmptyFiles()) // Drop file if no output was produced (e.g. no background.css)
+            .pipe(gulp.dest(destination))
+    })
 
-    function bundle() {
+    function bundle(callback) {
         let startTime = Date.now()
         b.bundle()
             .on('error', error => console.error(error.message))
@@ -99,10 +100,13 @@ function createBundle({entries, output, destination, cssOutput},
             .on('end', () => {
                 let time = (Date.now() - startTime) / 1000
                 console.log(`Bundled ${output} in ${time}s.`)
+                if (!watch) {
+                    callback()
+                }
             })
     }
 
-    bundle()
+    await pify(bundle)()
 }
 
 gulp.task('copyStaticFiles', () => {
@@ -113,34 +117,63 @@ gulp.task('copyStaticFiles', () => {
     }
 })
 
-gulp.task('lint', () => {
-    return gulp.src(['src/**/*.js', 'src/**/*.jsx'])
-        .pipe(eslint())
-        .pipe(eslint.format())
-        .pipe(eslint.results(results => {
-            console.log(`Total Errors: ${results.errorCount}`)
-        }))
+gulp.task('build-prod', ['copyStaticFiles'], async () => {
+    const ps = sourceFiles.map(filePath => createBundle({filePath, watch: false, production: true}))
+    await Promise.all(ps)
 })
 
-gulp.task('build-prod', ['copyStaticFiles', 'lint'], () => {
-    sourceFiles.forEach(bundle => createBundle(bundle, {watch: false, production: true}))
+gulp.task('build', ['copyStaticFiles'], async () => {
+    const ps = sourceFiles.map(filePath => createBundle({filePath, watch: false}))
+    await Promise.all(ps)
 })
 
-gulp.task('build', ['copyStaticFiles', 'lint'], () => {
-    sourceFiles.forEach(bundle => createBundle(bundle, {watch: false}))
+gulp.task('build-watch', ['copyStaticFiles'], async () => {
+    const ps = sourceFiles.map(filePath => createBundle({filePath, watch: true}))
+    await Promise.all(ps)
 })
 
-gulp.task('build-watch', ['copyStaticFiles'], () => {
-    sourceFiles.forEach(bundle => createBundle(bundle, {watch: true}))
-})
 
-gulp.task('lint-watch', ['lint'], () => {
-    gulp.watch(['src/**/*.js', 'src/**/*.jsx'])
-    .on('change', (file) => {
-        return gulp.src(file.path)
+// === Tasks for linting the source code ===
+
+const stylelintOptions = {
+    failAfterError: false,
+    reporters: [
+        {formatter: 'string', console: true},
+    ],
+}
+
+gulp.task('lint', async () => {
+    const eslintStream = gulp.src(['src/**/*.js', 'src/**/*.jsx'])
         .pipe(eslint())
         .pipe(eslint.format())
-    })
+        .pipe(eslint.results(results => {
+            // For clarity, also give some output when there are no errors.
+            if (results.errorCount === 0) {
+                console.log(`No eslint errors.\n`)
+            }
+        }))
+    await streamToPromise(eslintStream)
+
+    const stylelintStream = gulp.src(['src/**/*.css'])
+        .pipe(stylelint(stylelintOptions))
+    await streamToPromise(stylelintStream)
+})
+
+gulp.task('lint-watch', ['lint'], callback => {
+    gulp.watch(['src/**/*.js', 'src/**/*.jsx'])
+        .on('change', event => {
+            return gulp.src(event.path)
+                .pipe(eslint())
+                .pipe(eslint.format())
+        })
+
+    gulp.watch(['src/**/*.css'])
+        .on('change', event => {
+            return gulp.src(event.path)
+                .pipe(stylelint(stylelintOptions))
+        })
+
+    // Don't call callback, to wait forever.
 })
 
 gulp.task('watch', ['build-watch', 'lint-watch'])

package.json

@@ -8,13 +8,13 @@
     "url": "https://github.com/WebMemex/webmemex-extension.git"
   },
   "scripts": {
-    "build-prod": "gulp build-prod",
-    "build": "gulp build",
-    "watch": "gulp watch",
+    "build-prod": "rm -rf ./extension && gulp build-prod",
+    "build": "rm -rf ./extension && gulp build",
+    "watch": "rm -rf ./extension && gulp watch",
     "package": "gulp package",
-    "test": "npm run lint && jest --verbose --config=.jest-config.json",
-    "fx-run": "web-ext -s extension run --bc",
-    "lint": "gulp lint && stylelint '**/*.css'",
+    "test": "jest --verbose --config=.jest-config.json",
+    "firefox": "web-ext -s extension run --bc",
+    "lint": "gulp lint",
     "lint-fix": "eslint --ext js,jsx --fix ."
   },
   "dependencies": {
@@ -75,9 +75,13 @@
     "eslint-plugin-react": "^7.1.0",
     "eslint-plugin-standard": "^3.0.1",
     "gulp": "^3.9.1",
+    "gulp-add-src": "^0.2.0",
+    "gulp-clip-empty-files": "^0.1.2",
+    "gulp-concat-css": "^2.3.0",
     "gulp-eslint": "^4.0.0",
     "gulp-identity": "^0.1.0",
     "gulp-run": "^1.7.1",
+    "gulp-stylelint": "^4.0.0",
     "gulp-uglify": "^3.0.0",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^20.0.4",
@@ -93,6 +97,7 @@
     "redux-devtools": "^3.4.0",
     "redux-devtools-dock-monitor": "^1.1.2",
     "redux-devtools-log-monitor": "^1.3.0",
+    "stream-to-promise": "^2.2.0",
     "stylelint": "^7.11.1",
     "stylelint-config-standard": "^16.0.0",
     "uglifyify": "^4.0.1",

src/background.js

@@ -5,7 +5,7 @@ import manifest from '../extension/manifest.json'
 
 function openOverview() {
     browser.tabs.create({
-        url: '/overview/overview.html',
+        url: '/overview.html',
     })
 }
 

src/local-page.html

@@ -0,0 +1,7 @@
+<meta charset="utf-8" />
+<link rel="stylesheet" href="/lib/semantic-ui/semantic.min.css" />
+<link rel="stylesheet" href="local-page/local-page.css" />
+
+<div id="bar" />
+<script src="/lib/browser-polyfill.js"></script>
+<script src="/local-page/local-page.js"></script>

src/page-viewer/index.js → src/local-page/index.js

@@ -6,7 +6,7 @@ export function hrefForLocalPage({page}) {
         return undefined
     }
     const pageId = page._id
-    const url = `/page-viewer/localpage.html?page=${encodeURIComponent(pageId)}`
+    const url = `/local-page.html?page=${encodeURIComponent(pageId)}`
     const hash = (page.url && page.url.split('#')[1])
     const href = (hash !== undefined) ? url + '#' + hash : url
     return href

extension/manifest.json → src/manifest.json

@@ -11,7 +11,7 @@
     }],
     "browser_action": {
         "default_title": "WebMemex",
-        "default_popup": "/popup/popup.html"
+        "default_popup": "/popup.html"
     },
     "commands": {
         "openOverview": {

src/omnibar.js

@@ -3,7 +3,7 @@ import fromPairs from 'lodash/fp/fromPairs'
 import escapeHtml from 'lodash/fp/escape'
 
 import { filterVisitsByQuery } from 'src/search'
-import { hrefForLocalPage } from 'src/page-viewer'
+import { hrefForLocalPage } from 'src/local-page'
 import niceTime from 'src/util/nice-time'
 
 
@@ -95,7 +95,7 @@ const acceptInput = (text, disposition) => {
         url = suggestionToUrl[text]
     } else {
         // Treat input as search query, open the search
-        url = `/overview/overview.html?q=${encodeURIComponent(text)}`
+        url = `/overview.html?q=${encodeURIComponent(text)}`
     }
     // Open it in the place the browser requests us to.
     switch (disposition) {

extension/overview/overview.html → src/overview.html

@@ -1,7 +1,7 @@
 <meta charset="utf-8" />
 <title>📇 Memory overview</title>
-<link rel="stylesheet" href="style.css" />
+<link rel="stylesheet" href="/overview/overview.css" />
 
 <div id="app" />
 <script src="/lib/browser-polyfill.js"></script>
-<script src="overview.js"></script>
+<script src="/overview/overview.js"></script>

src/overview/components/VisitAsListItem.jsx

@@ -5,7 +5,7 @@ import { connect } from 'react-redux'
 import { Button, Popup, Icon } from 'semantic-ui-react'
 import classNames from 'classnames'
 
-import { hrefForLocalPage } from 'src/page-viewer'
+import { hrefForLocalPage } from 'src/local-page'
 import niceTime from 'src/util/nice-time'
 
 import ImgFromPouch from './ImgFromPouch'