Assets #597

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
       - checkout
       - run: echo $PYVERSION > ver.txt
       - restore_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
       - attach_workspace:
           at: ~/dash
       - store_artifacts:
@@ -34,15 +34,15 @@ jobs:
       - checkout
       - run: echo $PYVERSION > ver.txt
       - restore_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
       - run:
           name: ️️🏗️ pip dev requirements
           command: |
             sudo pip install --upgrade virtualenv
             python -m venv venv || virtualenv venv && . venv/bin/activate
-            pip install -e . -r requires-install.txt -r requires-ci.txt -r requires-testing.txt --quiet
+            pip install -e . -r requires-install.txt -r requires-dev.txt -r requires-testing.txt --quiet
       - save_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
           paths:
               - "venv"
       - run:
@@ -86,23 +86,23 @@ jobs:
       - checkout
       - run: echo $PYVERSION > ver.txt
       - restore_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
       - run:
           name: ️️🏗️ pip dev requirements
           command: |
             sudo pip install --upgrade virtualenv
             python -m venv venv || virtualenv venv && . venv/bin/activate
-            pip install -e . -r requires-install.txt -r requires-ci.txt -r requires-testing.txt --quiet
+            pip install -e . -r requires-install.txt -r requires-dev.txt -r requires-testing.txt --quiet
       - save_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
           paths:
               - "venv"
       - run:
           name: ️️🏗️ build core
           command: |
               . venv/bin/activate && pip install --upgrade -e . --quiet && mkdir packages
               python setup.py sdist && mv dist/* packages/
-              cd dash-renderer && npm run build && python setup.py sdist && mv dist/* ../packages/ && cd ..
+              cd dash-renderer && renderer build && python setup.py sdist && mv dist/* ../packages/ && cd ..
               git clone --depth 1 https://github.com/plotly/dash-core-components.git
               cd dash-core-components && npm install --ignore-scripts && npm run build && python setup.py sdist && mv dist/* ../packages/  && cd ..
               git clone --depth 1 https://github.com/plotly/dash-renderer-test-components
@@ -137,15 +137,15 @@ jobs:
       - checkout
       - run: echo $PYVERSION > ver.txt
       - restore_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
       - run:
           name: ️️🏗️ pip dev requirements
           command: |
             sudo pip install --upgrade virtualenv
             python -m venv venv || virtualenv venv && . venv/bin/activate
-            pip install -e . -r requires-install.txt -r requires-ci.txt -r requires-testing.txt --quiet
+            pip install -e . -r requires-install.txt -r requires-dev.txt -r requires-testing.txt --quiet
       - save_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
           paths:
               - "venv"
       - run:
@@ -187,7 +187,7 @@ jobs:
       - checkout
       - run: echo $PYVERSION > ver.txt
       - restore_cache:
-          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-ci.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
+          key: dep-{{ checksum "ver.txt" }}-{{ checksum "requires-dev.txt" }}-{{ checksum "requires-install.txt" }}-{{ checksum "requires-testing.txt" }}
       - attach_workspace:
           at: ~/dash
       - run:

CONTRIBUTING.md

@@ -12,11 +12,11 @@ $ python3 -m venv .venv/dev
 # activate the virtualenv (on windows .venv\dev\scripts\activate)
 $ . .venv/dev/bin/activate
 # install dash and dependencies
-$ pip install -e .[testing,ci]  # in some shells you need \ to escape []
+$ pip install -e .[testing,dev]  # in some shells you need \ to escape []
 $ cd dash-renderer
 # build renderer bundles, this will build all bundles from source code
 # the only true source of npm version is defined in package.json
-$ npm run build
+$ npm run build  # or `renderer build`
 # install dash-renderer for development
 $ pip install -e .
 # you should see both dash and dash-renderer are pointed to local source repos
@@ -29,14 +29,16 @@ $ pip list | grep dash
 
 If you want to contribute or simply dig deeper into Dash, we encourage you to play and taste it. This is the most efficient way to learn and understand everything under the hood.
 
-For contributors with a primarily  **Python**  or  **R**  background, this section might help you understand more details about developing and debugging in Javascript world.
+For contributors with a primarily **Python** or **R** background, this section might help you understand more details about developing and debugging in Javascript world.
 
 As of Dash 1.2, the renderer bundle and its peer dependencies can be packed and generated from the source code. The `dash-renderer\package.json` file is the one version of the truth for dash renderer version and npm dependencies. A build tool `renderer`, which is a tiny Python script installed by Dash as a command-line tool, has a few commands:
 
-1.  `renderer npm`  installs all the npm modules using this  `package.json`  files. Note that the  `package-lock.json`  file is the computed reference product for the versions defined with tilde(~) or caret(^) syntax in npm.
-2.  `renderer bundles` parses the locked version JSON, copies all the peer dependencies into dash_renderer folder, bundles the renderer assets, and generates an `__init__.py` to map all the resources. There are also a list of helpful `scripts` defined in `package.json` you might need to do some handy tasks like linting, syntax format with prettier, etc.
-3.  `renderer digest` computes the content hash of each asset in `dash_renderer` folder, prints out the result in logs, and dumps into a JSON file `digest.json`. Use this when you have a doubt about the current assets in `dash_renderer`, and compare it with previous result in one shot by this command.
-4.  `renderer watch` runs the webpack in watch mode, so any source code change triggers a rebuild. Use this if you are actively updating renderer code and you want to test your changes immediately.
+1. `renderer clean` deletes all the previously generated assets by this same tool.
+2.  `renderer npm`  installs all the npm modules using this  `package.json`  files. Note that the  `package-lock.json`  file is the computed reference product for the versions defined with tilde(~) or caret(^) syntax in npm.
+3.  `renderer bundles` parses the locked version JSON, copies all the peer dependencies into dash_renderer folder, bundles the renderer assets, and generates an `__init__.py` to map all the resources. There are also a list of helpful `scripts` property defined in `package.json` you might need to do some handy tasks like linting, syntax format with prettier, etc.
+4.  `renderer digest` computes the content hash of each asset in `dash_renderer` folder, prints out the result in logs, and dumps into a JSON file `digest.json`. Use this when you have a doubt about the current assets in `dash_renderer`, and compare it with previous result in one shot by this command.
+5.  `renderer build` runs 1, 2, 3, 4 in sequence as a complete build process from scratch.
+6.  `renderer watch` runs the webpack in watch mode, so any source code change triggers a rebuild. Use this if you are actively updating renderer code and you want to test your changes immediately.
 
 When a change in renderer code doesn't reflect in your browser as expected, this could be: confused bundle generation, caching issue in a browser, python package not in `editable` mode, etc. The new tool reduces the risk of bundle assets by adding the digest to help compare asset changes.
 

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017 Plotly, Inc
+Copyright (c) 2019 Plotly, Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

dash-renderer/package.json

@@ -9,9 +9,9 @@
     "prepublish": "npm test && npm run build && npm run build:dev",
     "publish-all": "npm publish && echo \"twine!!!\"",
     "publish-pypi": "npm run prepublish && python setup.py sdist upload",
-    "build:renderer": "webpack --mode development && webpack --mode production",
+    "build:webpack": "webpack --mode development && webpack --mode production",
     "build:dev": "webpack --mode development --watch",
-    "build": "renderer npm && renderer bundles",
+    "build": "renderer build",
     "start": "webpack-serve ./webpack.serve.config.js",
     "format": "prettier --config .prettierrc --write \"src/**/*.js\"",
     "format:test": "prettier --config .prettierrc \"src/**/*.js\" --list-different",

dash/_utils.py

@@ -153,11 +153,9 @@ def job(msg=""):
     def wrapper(func):
         @wraps(func)
         def _wrapper(*args, **kwargs):
-            logger.info(" 🏗️  %s ️️🏗️️ ", msg)
+            logger.info("🏗️  [%s] 🏗️️  - %s", func.__name__, msg)
             res = func(*args, **kwargs)
-            logger.info("::: 🍻🍻🍻 job done 🍻🍻🍻 :::")
+            logger.info("::: 🍻🍻🍻 [%s] job done 🍻🍻🍻 :::", func.__name__)
             return res
-
         return _wrapper
-
     return wrapper

dash/development/build_process.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+import sys
+import json
+import string
+import shutil
+import logging
+import coloredlogs
+import fire
+
+from .._utils import run_command_with_process, compute_md5, job
+
+logger = logging.getLogger(__name__)
+coloredlogs.install(
+    fmt="%(asctime)s,%(msecs)03d %(levelname)s - %(message)s",
+    datefmt="%H:%M:%S",
+)
+
+
+class BuildProcess(object):
+    def __init__(self, main, deps_info):
+        self.logger = logger
+        self.main = main
+        self.deps_info = deps_info
+        self.npm_modules = self._concat(self.main, "node_modules")
+        self.package_lock = self._concat(self.main, "package-lock.json")
+        self.package = self._concat(self.main, "package.json")
+        self._parse_package(path=self.package)
+        self.asset_paths = (self.build_folder, self.npm_modules)
+
+    def _parse_package(self, path):
+        with open(path, "r") as fp:
+            package = json.load(fp)
+            self.version = package["version"]
+            self.name = package["name"]
+            self.build_folder = self._concat(
+                self.main, self.name.replace("-", "_")
+            )
+            self.deps = package["dependencies"]
+
+    @staticmethod
+    def _concat(*paths):
+        return os.path.realpath(
+            os.path.sep.join((path for path in paths if path))
+        )
+
+    @staticmethod
+    def _clean_path(path):
+        if os.path.exists(path):
+            logger.warning("🚨 %s already exists, remove it!", path)
+            try:
+                if os.path.isfile(path):
+                    os.remove(path)
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+            except OSError:
+                sys.exit(1)
+        else:
+            logger.warning("🚨 %s doesn't exist, no action taken", path)
+
+    @job("clean all the previous assets generated by build tool")
+    def clean(self):
+        for path in self.asset_paths:
+            self._clean_path(path)
+
+    @job("run `npm i --ignore-scripts`")
+    def npm(self):
+        """job to install npm packages"""
+        os.chdir(self.main)
+        self._clean_path(self.package_lock)
+        run_command_with_process("npm i --ignore-scripts")
+
+    @job("build the renderer in dev mode")
+    def watch(self):
+        os.chdir(self.main)
+        os.system("npm run build:dev")
+
+    @job("run the whole building process in sequence")
+    def build(self):
+        self.clean()
+        self.npm()
+        self.bundles()
+        self.digest()
+
+    @job("compute the hash digest for assets")
+    def digest(self):
+        copies = tuple(
+            _
+            for _ in os.listdir(self.build_folder)
+            if os.path.splitext(_)[-1] in {".js", ".map"}
+        )
+        logger.info("bundles in %s %s", self.build_folder, copies)
+
+        payload = {self.name: self.version}
+        for copy in copies:
+            payload["MD5 ({})".format(copy)] = compute_md5(
+                self._concat(self.build_folder, copy)
+            )
+
+        with open(self._concat(self.main, "digest.json"), "w") as fp:
+            json.dump(
+                payload, fp, sort_keys=True, indent=4, separators=(",", ":")
+            )
+        logger.info(
+            "bundle digest in digest.json:\n%s",
+            json.dumps(payload, sort_keys=True, indent=4),
+        )
+
+    @job("copy and generate the bundles")
+    def bundles(self):
+        if not os.path.exists(self.build_folder):
+            try:
+                os.makedirs(self.build_folder)
+            except OSError:
+                logger.exception(
+                    "🚨 having issues manipulating %s", self.build_folder
+                )
+                sys.exit(1)
+
+        self._parse_package(self.package_lock)
+
+        getattr(self, "_bundles_extra", lambda: None)()
+
+        versions = {
+            "version": self.version,
+            "package": self.name.replace(" ", "_").replace("-", "_"),
+        }
+        for name, subfolder, filename, target in self.deps_info:
+            version = self.deps[name]["version"]
+            versions[name.replace("-", "").replace(".", "")] = version
+
+            logger.info("copy npm dependency => %s", filename)
+            ext = "min.js" if "min" in filename.split(".") else "js"
+            target = (
+                target.format(version)
+                if target
+                else "{}@{}.{}".format(name, version, ext)
+            )
+            shutil.copyfile(
+                self._concat(self.npm_modules, name, subfolder, filename),
+                self._concat(self.build_folder, target),
+            )
+
+        logger.info("run `npm run build:webpack`")
+        os.chdir(self.main)
+        run_command_with_process("npm run build:webpack")
+
+        logger.info("generate the `__init__.py` from template and verisons")
+        with open(self._concat(self.main, "init.template")) as fp:
+            t = string.Template(fp.read())
+
+        with open(self._concat(self.build_folder, "__init__.py"), "w") as fp:
+            fp.write(t.safe_substitute(versions))
+
+
+class Renderer(BuildProcess):
+    def __init__(self):
+        # dash-renderer's path is binding with the dash folder hierarchy
+        super(Renderer, self).__init__(
+            self._concat(
+                os.path.dirname(__file__),
+                os.pardir,
+                os.pardir,
+                "dash-renderer",
+            ),
+            (
+                ("react", "umd", "react.production.min.js", None),
+                ("react", "umd", "react.development.js", None),
+                ("react-dom", "umd", "react-dom.production.min.js", None),
+                ("react-dom", "umd", "react-dom.development.js", None),
+                ("prop-types", None, "prop-types.min.js", None),
+                ("prop-types", None, "prop-types.js", None),
+            ),
+        )
+
+
+def renderer():
+    fire.Fire(Renderer)