partial request deduping

.eslintrc

@@ -78,7 +78,7 @@
         "no-throw-literal": [ 2 ],
         "no-unused-expressions": [ 2 ],
 
-        "no-warning-comments": [ 1 ],
+        "no-warning-comments": [ 0 ],
         "no-with": [ 2 ],
         "radix": [ 2 ],
         "wrap-iife": [ 2 ],

lib/get/onMissing.js

@@ -1,5 +1,7 @@
 var support = require("./util/support");
 var fastCopy = support.fastCopy;
+var fastCat = support.fastCat;
+var arraySlice = require("./../support/array-slice");
 
 module.exports = function onMissing(model, path, depth,
                                     outerResults, requestedPath,
@@ -8,6 +10,7 @@ module.exports = function onMissing(model, path, depth,
     if (!outerResults.requestedMissingPaths) {
         outerResults.requestedMissingPaths = [];
         outerResults.optimizedMissingPaths = [];
+        outerResults.depthDifferences = [];
     }
 
     if (depth < path.length) {
@@ -31,15 +34,13 @@ module.exports = function onMissing(model, path, depth,
 
 function concatAndInsertMissing(model, remainingPath, depth, requestedPath,
                                 optimizedPath, optimizedLength, results) {
+    results.requestedMissingPaths[results.requestedMissingPaths.length] =
+        fastCat(arraySlice(requestedPath, 0, depth), remainingPath);
 
-    // TODO: Performance.
-    results.requestedMissingPaths.push(
-        requestedPath.
-            slice(0, depth).
-            concat(remainingPath));
+    results.optimizedMissingPaths[results.optimizedMissingPaths.length] =
+        fastCat(arraySlice(optimizedPath, 0, optimizedLength), remainingPath);
 
-    results.optimizedMissingPaths.push(
-        optimizedPath.slice(0, optimizedLength).concat(remainingPath));
+    results.depthDifferences[results.depthDifferences.length] = depth - optimizedLength;
 }
 
 function isEmptyAtom(atom) {

lib/request/GetRequestV2.js

@@ -116,13 +116,16 @@ GetRequestV2.prototype = {
     /**
      * Attempts to add paths to the outgoing request.  If there are added
      * paths then the request callback will be added to the callback list.
+     * Handles adding partial paths as well
      *
-     * @returns {Array} - the remaining paths in the request.
+     * @returns {Array} - whether new requested paths were inserted in this
+     *                    request, the remaining paths that could not be added,
+     *                    and disposable for the inserted requested paths.
      */
-    add: function(requested, optimized, callback) {
+    add: function(requested, optimized, depthDifferences, callback) {
         // uses the length tree complement calculator.
         var self = this;
-        var complementTuple = complement(requested, optimized, self._pathMap);
+        var complementTuple = complement(requested, optimized, depthDifferences, self._pathMap);
         var optimizedComplement;
         var requestedComplement;
 
@@ -137,10 +140,9 @@ GetRequestV2.prototype = {
         var inserted = false;
         var disposable = false;
 
-        // If the out paths is less than the passed in paths, then there
-        // has been an intersection and the complement has been returned.
-        // Therefore, this can be deduped across requests.
-        if (optimizedComplement.length < optimized.length) {
+        // If we found an intersection, then just add new callback
+        // as one of the dependents of that request
+        if (complementTuple && complementTuple[0].length) {
             inserted = true;
             var idx = self._callbacks.length;
             self._callbacks[idx] = callback;

lib/request/RequestQueueV2.js

@@ -43,7 +43,7 @@ RequestQueueV2.prototype = {
      * @param {Array} optimizedPaths -
      * @param {Function} cb -
      */
-    get: function(requestedPaths, optimizedPaths, cb) {
+    get: function(requestedPaths, optimizedPaths, depthDifferences, cb) {
         var self = this;
         var disposables = [];
         var count = 0;
@@ -64,7 +64,7 @@ RequestQueueV2.prototype = {
             // if possible.
             if (request.sent) {
                 var results = request.add(
-                    rRemainingPaths, oRemainingPaths, refCountCallback);
+                    rRemainingPaths, oRemainingPaths, depthDifferences, refCountCallback);
 
                 // Checks to see if the results were successfully inserted
                 // into the outgoing results.  Then our paths will be reduced

lib/request/complement.js

@@ -1,50 +1,213 @@
 var hasIntersection = require("falcor-path-utils").hasIntersection;
 var arraySlice = require("./../support/array-slice");
+var arrayConcat = require("./../support/array-concat");
+var iterateKeySet = require("falcor-path-utils").iterateKeySet;
 
 /**
- * creates the complement of the requested and optimized paths
- * based on the provided tree.
+ * Figures out what paths in requested pathsets can be
+ * deduped based on existing optimized path tree provided.
  *
- * If there is no complement then this is just a glorified
- * array copy.
+ * ## no deduping possible:
+ *
+ * if no existing requested sub tree at all for path,
+ * just add the entire path to complement.
+ *
+ * ## fully deduped:
+ *
+ * if required path is a complete subset of given sub tree,
+ * just add the entire path to intersection
+ *
+ * ## partial deduping:
+ *
+ * if some part of path, when ranges are expanded, is a subset
+ * of given sub tree, then add only that part to intersection,
+ * and all other parts of this path to complement
+ *
+ * To keep `depth` argument be a valid index for optimized path (`oPath`),
+ * either requested or optimized path is sent in pre-initialized with
+ * some items so that their remaining length matches exactly, keeping
+ * remaining ranges in those pathsets 1:1 in correspondence
+ *
+ * Note that positive `depthDiff` value means that requested path is
+ * longer than optimized path, and we need to pre-initialize current
+ * requested path with that many offset items, so that their remaining
+ * length matches. Similarly, negative `depthDiff` value means that
+ * optimized path is longer, and we pre-initialize optimized path with
+ * those many items. Note that because of the way requested and
+ * optimized paths are accumulated from what user requested in model.get
+ * (see onMissing.js), it is not possible for the pre-initialized paths
+ * to have any ranges in them.
+ *
+ * `intersectionData` is:
+ * [ requestedIntersection, optimizedComplement, requestedComplement ]
+ * where `requestedIntersection` is matched requested paths that can be
+ * deduped, `optimizedComplement` is missing optimized paths, and
+ * `requestedComplement` is requested counterparts of those missing
+ * optimized paths
  */
-module.exports = function complement(requested, optimized, tree) {
+module.exports = function complement(requested, optimized, depthDifferences, tree) {
     var optimizedComplement = [];
     var requestedComplement = [];
     var requestedIntersection = [];
     var intersectionLength = -1, complementLength = -1;
-    var intersectionFound = false;
 
     for (var i = 0, len = optimized.length; i < len; ++i) {
-        // If this does not intersect then add it to the output.
-        var path = optimized[i];
-        var subTree = tree[path.length];
+        var oPath = optimized[i];
+        var rPath = requested[i];
+        var depthDiff = depthDifferences[i];
+        var subTree = tree[oPath.length];
 
-        // If there is no subtree to look into or there is no intersection.
-        if (!subTree || !hasIntersection(subTree, path, 0)) {
-
-            if (intersectionFound) {
-                optimizedComplement[++complementLength] = path;
-                requestedComplement[complementLength] = requested[i];
-            }
-        } else {
-            // If there has been no intersection yet and
-            // i is bigger than 0 (meaning we have had only complements)
-            // then we need to update our complements to match the current
-            // reality.
-            if (!intersectionFound && i > 0) {
-                requestedComplement = arraySlice(requested, 0, i);
-                optimizedComplement = arraySlice(optimized, 0, i);
-            }
+        // no deduping possible
+        if (!subTree) {
+            optimizedComplement[++complementLength] = oPath;
+            requestedComplement[complementLength] = rPath;
+            continue;
+        }
+        // fully deduped
+        if (hasIntersection(subTree, oPath, 0)) {
+            requestedIntersection[++intersectionLength] = rPath;
+            continue;
+        }
 
-            requestedIntersection[++intersectionLength] = requested[i];
-            intersectionFound = true;
+        // partial deduping
+        var intersectionData = findPartialIntersections(
+            rPath,
+            oPath,
+            subTree,
+            depthDiff < 0 ? -depthDiff : 0,
+            depthDiff > 0 ? arraySlice(rPath, 0, depthDiff) : [],
+            depthDiff < 0 ? arraySlice(oPath, 0, -depthDiff) : [],
+            depthDiff);
+        for (var j = 0, jLen = intersectionData[0].length; j < jLen; ++j) {
+            requestedIntersection[++intersectionLength] = intersectionData[0][j];
+        }
+        for (var k = 0, kLen = intersectionData[1].length; k < kLen; ++k) {
+            optimizedComplement[++complementLength] = intersectionData[1][k];
+            requestedComplement[complementLength] = intersectionData[2][k];
         }
     }
 
-    if (!intersectionFound) {
+    if (!requestedIntersection.length) {
         return null;
     }
-
-    return [requestedIntersection, optimizedComplement, requestedComplement ];
+    return [requestedIntersection, optimizedComplement, requestedComplement];
 };
+
+/**
+ * Recursive function to calculate intersection and complement paths in 2 given
+ * pathsets at a given depth
+ * Parameters:
+ *  - `requestedPath`: full requested path (can include ranges)
+ *  - `optimizedPath`: corresponding optimized path (can include ranges)
+ *  - `currentTree`: path map for in-flight request, against which to dedupe
+ *  - `depth`: index of optimized path that we are trying to match with `currentTree`
+ *  - `rCurrentPath`: current accumulated requested path by previous recursive
+ *                    iterations. Could also have been pre-initialized as stated
+ *                    above.
+ *                    This path cannot contain ranges, instead contains a key
+ *                    from the range, representing one of the individual paths
+ *                    in `requestedPath` pathset
+ *  - `oCurrentPath`: corresponding accumulated optimized path, to be matched
+ *                    with `currentTree`. Could have been pre-initialized.
+ *                    Cannot contain ranges, instead contains a key from the
+ *                    range at given `depth` in `optimizedPath`
+ *  - `depthDiff`: difference in length between `requestedPath` and `optimizedPath`
+ *
+ *  Example scenario:
+ *      - requestedPath: ['lolomo', 0, 0, 'tags', { from: 0, to: 2 }]
+ *      - optimizedPath: ['videosById', 11, 'tags', { from: 0, to: 2 }]
+ *      - currentTree: { videosById: 11: { tags: { 0: null, 1: null }}}
+ *      // since requested path is longer, optimized path index starts from depth 0
+ *      // and accumulated requested path starts pre-initialized (rCurrentPath)
+ *      - depth: 0
+ *      - rCurrentPath: ['lolomo']
+ *      - oCurrentPath: []
+ *      - depthDiff: 1
+ */
+function findPartialIntersections(requestedPath, optimizedPath, currentTree, depth, rCurrentPath, oCurrentPath, depthDiff) {
+    var intersections = [];
+    var rComplementPaths = [];
+    var oComplementPaths = [];
+    // iterate over optimized path, looking for deduping opportunities
+    for (; depth < optimizedPath.length; ++depth) {
+        var key = optimizedPath[depth];
+        var keyType = typeof key;
+
+        // if range key is found, start inner loop to iterate over all keys in range
+        // and add intersections and complements from each iteration separately.
+        // range keys branch-out like this, providing individual deduping
+        // opportunities for each inner key
+        if (key && keyType === "object") {
+            var note = {};
+            var innerKey = iterateKeySet(key, note);
+
+            while (!note.done) {
+                var nextTree = currentTree[innerKey];
+                if (nextTree === undefined) {
+                    // if no next sub tree exists for an inner key, it's a dead-end
+                    // and we can add this to complement paths
+                    var oPath = oCurrentPath.concat(
+                        innerKey,
+                        arraySlice(
+                            optimizedPath,
+                            depth + 1));
+                    oComplementPaths[oComplementPaths.length] = oPath;
+                    var rPath = rCurrentPath.concat(
+                        innerKey,
+                        arraySlice(
+                            requestedPath,
+                            depth + 1 + depthDiff));
+                    rComplementPaths[rComplementPaths.length] = rPath;
+                } else if (depth === optimizedPath.length - 1) {
+                    // reaching the end of optimized path means that we found a
+                    // corresponding node in the path map tree every time,
+                    // so add current path to successful intersections
+                    intersections[intersections.length] = arrayConcat(rCurrentPath, [innerKey]);
+                } else {
+                    // otherwise keep trying to find further partial deduping
+                    // opportunities in the remaining path!
+                    var intersectionData = findPartialIntersections(
+                        requestedPath,
+                        optimizedPath,
+                        nextTree,
+                        depth + 1,
+                        arrayConcat(rCurrentPath, [innerKey]),
+                        arrayConcat(oCurrentPath, [innerKey]),
+                        depthDiff);
+                    for (var j = 0, jLen = intersectionData[0].length; j < jLen; ++j) {
+                        intersections[intersections.length] = intersectionData[0][j];
+                    }
+                    for (var k = 0, kLen = intersectionData[1].length; k < kLen; ++k) {
+                        oComplementPaths[oComplementPaths.length] = intersectionData[1][k];
+                        rComplementPaths[rComplementPaths.length] = intersectionData[2][k];
+                    }
+                }
+                innerKey = iterateKeySet(key, note);
+            }
+            break;
+        }
+
+        // for simple keys, we don't need to branch out. looping over `depth`
+        // here instead of recursion, for performance
+        currentTree = currentTree[key];
+        oCurrentPath[oCurrentPath.length] = optimizedPath[depth];
+        rCurrentPath[rCurrentPath.length] = requestedPath[depth + depthDiff];
+
+        if (currentTree === undefined) {
+            // if dead-end, add this to complements
+            oComplementPaths[oComplementPaths.length] =
+                arrayConcat(oCurrentPath, arraySlice(optimizedPath, depth + 1));
+            rComplementPaths[rComplementPaths.length] =
+                arrayConcat(rCurrentPath, arraySlice(requestedPath, depth + depthDiff + 1));
+            break;
+        } else if (depth === optimizedPath.length - 1) {
+            // if reach end of optimized path successfully, add to intersections
+            intersections[intersections.length] = rCurrentPath;
+        }
+        // otherwise keep going
+    }
+
+    // return accumulated intersection and complement pathsets
+    return [intersections, oComplementPaths, rComplementPaths];
+}
+

lib/response/get/checkCacheAndReport.js

@@ -6,6 +6,27 @@ var getWithPathsAsPathMap = gets.getWithPathsAsPathMap;
  * Checks cache for the paths and reports if in progressive mode.  If
  * there are missing paths then return the cache hit results.
  *
+ * Return value (`results`) stores missing path information as 3 index-linked arrays:
+ * `requestedMissingPaths` holds requested paths that were not found in cache
+ * `optimizedMissingPaths` holds optimized versions of requested paths
+ * `depthDifferences` holds the difference in length of requested and optimized paths
+ *
+ * Note that requestedMissingPaths is not necessarily the list of paths requested by
+ * user in model.get. It does not contain those paths that were found in
+ * cache. It also breaks some path sets out into separate paths, those which
+ * resolve to different optimized lengths after walking through any references in
+ * cache.
+ * This helps maintain a 1:1 correspondence between requested and optimized missing,
+ * as well as their depth differences (or, length offsets).
+ *
+ * Example: Given cache: `{ lolomo: { 0: $ref('vid'), 1: $ref('a.b.c.d') }}`,
+ * `model.get('lolomo[0..2].name').subscribe()` will result in the following
+ * corresponding values:
+ *    index   requestedMissingPaths   optimizedMissingPaths         depthDifferences
+ *      0     ['lolomo', 0, 'name']   ['vid', 'name']                   1
+ *      1     ['lolomo', 1, 'name']   ['a', 'b', 'c', 'd', 'name']     -2
+ *      2     ['lolomo', 2, 'name']   ['lolomo', 2, 'name']             0
+ *
  * @param {Model} model - The model that the request was made with.
  * @param {Array} requestedMissingPaths -
  * @param {Boolean} progressive -
@@ -14,6 +35,8 @@ var getWithPathsAsPathMap = gets.getWithPathsAsPathMap;
  * @param {Function} onError -
  * @param {Function} onCompleted -
  * @param {Object} seed - The state of the output
+ * @returns {Object} results -
+ *
  * @private
  */
 module.exports = function checkCacheAndReport(model, requestedPaths, observer,