diff --git a/lib/styleToFilters.js b/lib/styleToFilters.js index e02b01b..1d707f4 100644 --- a/lib/styleToFilters.js +++ b/lib/styleToFilters.js @@ -1,3 +1,6 @@ +// import {isExpression} from ; +let styleSpec = require('@mapbox/mapbox-gl-style-spec'); + /** * Takes optimized filter object from shaver.styleToFilters and returns c++ filters for shave. * @function styleToFilters @@ -18,7 +21,7 @@ function styleToFilters(style) { var layers = {}; // Store layers and filters used in style - if (style.layers) { + if (style && style.layers) { for (var i = 0; i < style.layers.length; i++) { var layerName = style.layers[i]['source-layer']; if (layerName) { @@ -42,13 +45,134 @@ function styleToFilters(style) { layers[layerName].minzoom = style.layers[i].minzoom || 0; layers[layerName].maxzoom = style.layers[i].maxzoom || 22; } + + // Collect the used properties + // 1. from paint, layout, and filter + layers[layerName].properties = layers[layerName].properties || []; + ['paint', 'layout'].forEach(item => { + let itemObject = style.layers[i][item]; + itemObject && getPropertyFromLayoutAndPainter(itemObject, layers[layerName].properties); + }); + // 2. from filter + if (style.layers[i].filter) { + getPropertyFromFilter(style.layers[i].filter, layers[layerName].properties); + } } } } + // remove duplicate propertys and fix choose all the propertys in layers[i].properties + Object.keys(layers).forEach(layerId => { + let properties = layers[layerId].properties; + if (properties.indexOf(true) !== -1) { + layers[layerId].properties = true; + } else { + let unique = {}; + properties.forEach(function(i) { + if (!unique[i]) { + unique[i] = true; + } + }); + layers[layerId].properties = Object.keys(unique); + } + }); return layers; } +function getPropertyFromFilter(filter, properties) { + if (styleSpec.expression.isExpression(filter)) { + getPropertyFromExpression(filter, properties); + } + + // Warning: Below code should put in to an else conditions, + // but since the `isExpression` can not tell it is a expression or filter syntax I put it outsied the else + // this could reduce the performance or cause some potential bugs, we must keep an eye on this. + + // else { + let subFilter = []; + for (let i = 0; i < filter.length; i++) { + if (typeof filter[i] === 'object' && filter[i] instanceof Array) { + subFilter.push(filter[i]); + } + } + + if (subFilter.length > 0) { + subFilter.forEach(sfilter => { + getPropertyFromFilter(sfilter, properties); + }) + } else { + if (filter.length >= 3 && typeof filter[1] === 'string') { + if (filter[1].indexOf('$') === -1) { + properties.push(filter[1]); + } + + } + } + // } +} + + +function getPropertyFromLayoutAndPainter(propertyObj, properties) { + Object.keys(propertyObj).forEach(key => { + let value = propertyObj[key]; + // TODO we still have outher situation: + // - special properties: `mapbox_clip_start`, `mapbox_clip_end` + if (typeof value === 'string') { + // if the value is string try to get property name from `xx{PropertyName}xx` like. + // the /{[^}]+}/ig return all the value like {xxx} + // eg 'a{hello}badfa' => ['{hello}'] + // eg 'a{hello}ba{world}dfa' => ['{hello}','{world}'] + let preProperties = value.match(/{[^}]+}/ig); + preProperties && preProperties.forEach(item => { + properties.push(item.slice(1, -1)); + }); + } else if (typeof value === 'object' && typeof value.property === 'string') { + // - legacy functions with `property` + properties.push(value.property); + } else { + // test isExpression from sytleSpec + if (styleSpec.expression.isExpression(value)) { + // TODO: now we implement this by ourself in vtshavem, we need to talk with ‘style spec’ member to see if there have a official method to get used property, to make this can be synchronized with the expression update. + getPropertyFromExpression(value, properties); + } else { + // otherwise continual loop; + getPropertyFromLayoutAndPainter(value, properties); + } + } + }) +} + + +function getPropertyFromExpression(exp, properties) { + // now we care about the expression like: + // ["get", string] not ["get", string, Object], + // ["has", string] not ["has", string, Object], + // ["properties"], + // ["feature-state", string] + if (exp instanceof Array) { + switch (exp[0]) { + case 'get': + case 'has': + if (typeof exp[1] === 'string' && !(exp[2] && typeof exp[2] === 'object')) { + properties.push(exp[1]); + } + break; + case 'feature-state': + properties.push(exp[1]); + break; + case 'properties': + properties.push(true); + break; + } + + exp.forEach(sub => { + if (sub instanceof Array) { + getPropertyFromExpression(sub, properties) + } + }) + } +} + module.exports = styleToFilters; // cli tool @@ -69,9 +193,9 @@ if (require.main === module) { }); } else { throw new Error('wrong number of arguments\n' + - 'Usage:\n' + - ' node ./lib/styles-to-filter.js ./fixtures/style.json\n' + - 'or\n' + - ' node ./lib/styles-to-filter.js < ./fixtures/style.json'); + 'Usage:\n' + + ' node ./lib/styles-to-filter.js ./fixtures/style.json\n' + + 'or\n' + + ' node ./lib/styles-to-filter.js < ./fixtures/style.json'); } -} +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a04b517..927f0ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,32 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@mapbox/jsonlint-lines-primitives": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@mapbox/jsonlint-lines-primitives/-/jsonlint-lines-primitives-2.0.2.tgz", + "integrity": "sha1-zlblOfg1UrWNENZy6k1vya3HsjQ=" + }, + "@mapbox/mapbox-gl-style-spec": { + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/@mapbox/mapbox-gl-style-spec/-/mapbox-gl-style-spec-13.3.0.tgz", + "integrity": "sha512-jEdKvDRPW95EBNRhpGuzZl6DpToDtQkJGlicz7QH8lWz+IxDfHUJV46yj+g8wgWGFiiTY3+xUrJKo6XrYjFRjw==", + "requires": { + "@mapbox/jsonlint-lines-primitives": "~2.0.2", + "@mapbox/unitbezier": "^0.0.0", + "csscolorparser": "~1.0.2", + "json-stringify-pretty-compact": "^1.2.0", + "minimist": "0.0.8", + "rw": "^1.3.3", + "sort-object": "^0.3.2" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + } + } + }, "@mapbox/mason-js": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/@mapbox/mason-js/-/mason-js-0.1.5.tgz", @@ -44,6 +70,11 @@ "integrity": "sha512-pEsfZyG4OMThlfFQbCte4gegvHUjxXCjz0KZ4Xk8NdOYTQBLflj6U8PL05RPAiuRAMAQNUUKJuL6qYZ5Y4kAWA==", "dev": true }, + "@mapbox/unitbezier": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz", + "integrity": "sha1-FWUb1VOme4WB+zmIEMmK2Go0Uk4=" + }, "@mapbox/vector-tile": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@mapbox/vector-tile/-/vector-tile-1.3.1.tgz", @@ -171,6 +202,11 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "csscolorparser": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/csscolorparser/-/csscolorparser-1.0.3.tgz", + "integrity": "sha1-s085HupNqPPpgjHizNjfnAQfFxs=" + }, "d3-queue": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/d3-queue/-/d3-queue-3.0.7.tgz", @@ -438,6 +474,11 @@ "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", "dev": true }, + "json-stringify-pretty-compact": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-1.2.0.tgz", + "integrity": "sha512-/11Pj1OyX814QMKO7K8l85SHPTr/KsFxHp8GE2zVa0BtJgGimDjXHfM3FhC7keQdWDea7+nXf+f1de7ATZcZkQ==" + }, "jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", @@ -463,7 +504,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, "minipass": { @@ -716,6 +757,11 @@ "glob": "^7.0.5" } }, + "rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha1-P4Yt+pGrdmsUiF700BEkv9oHT7Q=" + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -746,6 +792,25 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "sort-asc": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/sort-asc/-/sort-asc-0.1.0.tgz", + "integrity": "sha1-q3md9h/HPqCVbHnEtTHtHp53J+k=" + }, + "sort-desc": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/sort-desc/-/sort-desc-0.1.1.tgz", + "integrity": "sha1-GYuMDN6wlcRjNBhh45JdTuNZqe4=" + }, + "sort-object": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/sort-object/-/sort-object-0.3.2.tgz", + "integrity": "sha1-mODRme3kDgfGGoRAPGHWw7KQ+eI=", + "requires": { + "sort-asc": "^0.1.0", + "sort-desc": "^0.1.1" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", diff --git a/package.json b/package.json index 71b3f7b..0d8e1fe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@mapbox/vtshaver", - "version": "0.1.3", + "version": "0.2.0", "description": "Creates style-optimized vector tiles", "main": "./lib/index.js", "repository": { @@ -12,11 +12,13 @@ "install": "node-pre-gyp install --fallback-to-build", "docs": "npm run docs-cpp && npm run docs-js", "docs-cpp": "documentation build src/*.cpp --re --polyglot -f md -o API-CPP.md", - "docs-js": "documentation build lib/styleToFilters.js -f md -o API-JavaScript.md" + "docs-js": "documentation build lib/styleToFilters.js -f md -o API-JavaScript.md", + "build:dev": "make debug" }, "author": "Mapbox", "license": "ISC", "dependencies": { + "@mapbox/mapbox-gl-style-spec": "^13.3.0", "nan": "^2.11.1", "node-pre-gyp": "^0.12.0" }, diff --git a/scripts/coverage.sh b/scripts/coverage.sh index d0f20c1..ef63673 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -18,6 +18,5 @@ CXX_MODULE=$(./node_modules/.bin/node-pre-gyp reveal module --silent) export PATH=$(pwd)/mason_packages/.link/bin/:${PATH} llvm-profdata merge -output=code.profdata code-*.profraw llvm-cov report ${CXX_MODULE} -instr-profile=code.profdata -use-color -llvm-cov show ${CXX_MODULE} -instr-profile=code.profdata src/*.cpp -filename-equivalence -use-color -llvm-cov show ${CXX_MODULE} -instr-profile=code.profdata src/*.cpp -filename-equivalence -use-color --format html > /tmp/coverage.html +llvm-cov show ${CXX_MODULE} -instr-profile=code.profdata src/*.cpp -path-equivalence -use-color -format html > /tmp/coverage.html echo "open /tmp/coverage.html for HTML version of this report" diff --git a/scripts/sanitize.sh b/scripts/sanitize.sh index 4168cca..28c9344 100755 --- a/scripts/sanitize.sh +++ b/scripts/sanitize.sh @@ -24,6 +24,7 @@ SUPPRESSION_FILE="/tmp/leak_suppressions.txt" echo "leak:__strdup" > ${SUPPRESSION_FILE} echo "leak:v8::internal" >> ${SUPPRESSION_FILE} echo "leak:node::CreateEnvironment" >> ${SUPPRESSION_FILE} +echo "leak:node::Start" >> ${SUPPRESSION_FILE} echo "leak:node::Init" >> ${SUPPRESSION_FILE} export ASAN_SYMBOLIZER_PATH=$(pwd)/mason_packages/.link/bin/llvm-symbolizer export MSAN_SYMBOLIZER_PATH=$(pwd)/mason_packages/.link/bin/llvm-symbolizer diff --git a/src/filters.cpp b/src/filters.cpp index 1e05024..ec913e1 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -65,21 +65,27 @@ NAN_METHOD(Filters::New) { // get v8::Array of layers v8::Local layers_val = filters->GetPropertyNames(); + // LCOV_EXCL_START + // the GetPropertyNames returns an Array, so we don't need to test the v8 if (!layers_val->IsArray() || layers_val->IsNull() || layers_val->IsUndefined()) { Nan::ThrowError("layers must be an array and cannot be null or undefined"); return; } + // LCOV_EXCL_STOP auto layers = layers_val.As(); // Even if there layers_val is a string instead of an array, convert it to an array // Loop through each layer in the object and convert its filter to a mbgl::style::Filter - uint32_t length = layers->Length(); - for (uint32_t i = 0; i < length; ++i) { + std::uint32_t length = layers->Length(); + for (std::uint32_t i = 0; i < length; ++i) { // get v8::String containing layer name v8::Local layer_name_val = layers->Get(i); + // LCOV_EXCL_START + // the layer_name_val is the name of the object, since we have checked the layers->Length(), which means layers->Get(i) can not be null undefined or others if (!layer_name_val->IsString() || layer_name_val->IsNull() || layer_name_val->IsUndefined()) { Nan::ThrowError("layer name must be a string and cannot be null or undefined"); return; } + // LCOV_EXCL_STOP auto layer_name = layer_name_val.As(); // get v8::Object containing layer @@ -119,7 +125,6 @@ NAN_METHOD(Filters::New) { } // handle filters array const v8::Local layer_filter = layer->Get(Nan::New("filters").ToLocalChecked()); - // error handling in case filter value passed in from JS-world is somehow invalid if (layer_filter->IsNull() || layer_filter->IsUndefined()) { Nan::ThrowError("Filters is not properly constructed."); @@ -157,10 +162,43 @@ NAN_METHOD(Filters::New) { // insert the key/value into filters map // TODO(dane): what if we have duplicate source-layer filters? + // handle property array + const v8::Local layer_properties = layer->Get(Nan::New("properties").ToLocalChecked()); + if (layer_properties->IsNull() || layer_properties->IsUndefined()) { + Nan::ThrowError("Property-Filters is not properly constructed."); + return; + } + + // NOTICE: If a layer is styled, but does not have a property, the property value will equal [] + // NOTICE: If a property is true, that means we need to keep all the properties + filter_properties_type property; + if (layer_properties->IsArray()) { + // const auto propertyArray = layer_properties.As(); + v8::Handle propertyArray = v8::Handle::Cast(layer_properties); + std::uint32_t propertiesLength = propertyArray->Length(); + std::vector values; + values.reserve(propertiesLength); + for (std::uint32_t index = 0; index < propertiesLength; ++index) { + v8::Local property_value = propertyArray->Get(index); + Nan::Utf8String utf8_value(property_value); + int utf8_len = utf8_value.length(); + if (utf8_len > 0) { + values.emplace_back(*utf8_value, static_cast(utf8_len)); + } + } + property.first = list; + property.second = values; + } else if (layer_properties->IsBoolean() && layer_properties->IsTrue()) { + property.first = all; + property.second = {}; + } else { + Nan::ThrowTypeError("invalid filter value, must be an array or a boolean"); + return; + } + std::string source_layer = *v8::String::Utf8Value(layer_name->ToString()); - self->add_filter(std::move(source_layer), std::move(filter), minzoom, maxzoom); - // can find these via filters.find("x")->second + self->add_filter(std::move(source_layer), std::move(filter), std::move(property), minzoom, maxzoom); } } } diff --git a/src/filters.hpp b/src/filters.hpp index f7f338c..85b7355 100644 --- a/src/filters.hpp +++ b/src/filters.hpp @@ -10,9 +10,12 @@ class Filters : public Nan::ObjectWrap { public: using filter_value_type = mbgl::style::Filter; + using filter_properties_types = enum { all, + list }; + using filter_properties_type = std::pair>; using filter_key_type = std::string; // TODO: convert to data_view using zoom_type = double; - using filter_values_type = std::tuple; + using filter_values_type = std::tuple; using filters_type = std::map; // initializer @@ -23,9 +26,9 @@ class Filters : public Nan::ObjectWrap { static Nan::Persistent& constructor(); - void add_filter(filter_key_type&& key, filter_value_type&& filter, zoom_type minzoom, zoom_type maxzoom) { + void add_filter(filter_key_type&& key, filter_value_type&& filter, filter_properties_type&& properties, zoom_type minzoom, zoom_type maxzoom) { // add a new key/value pair, with the value equaling a tuple 'filter_values_type' defined above - filters.emplace(key, std::make_tuple(std::move(filter), minzoom, maxzoom)); + filters.emplace(key, std::make_tuple(std::move(filter), std::move(properties), minzoom, maxzoom)); } filters_type const& get_filters() const { diff --git a/src/shave.cpp b/src/shave.cpp index 9498b2d..6f4cb0f 100644 --- a/src/shave.cpp +++ b/src/shave.cpp @@ -324,7 +324,8 @@ static mbgl::FeatureType convertGeom(vtzero::GeomType geometry_type) { void filterFeatures(vtzero::tile_builder* finalvt, float zoom, vtzero::layer const& layer, - mbgl::style::Filter const& mbgl_filter_obj) { + mbgl::style::Filter const& mbgl_filter_obj, + Filters::filter_properties_type const& property_filter) { /** * TODOs: * - Instead of decoding/re-encoding, we'll want to add bytes...? @@ -333,6 +334,21 @@ void filterFeatures(vtzero::tile_builder* finalvt, **/ vtzero::layer_builder layer_builder{*finalvt, layer}; vtzero::property_mapper mapper{layer, layer_builder}; + + Filters::filter_properties_types const& property_filter_type = property_filter.first; + std::vector const& properties = property_filter.second; + + auto const& keytable = layer.key_table(); + std::vector props_by_index; + for (auto const& prop : properties) { + auto itr = std::find(keytable.begin(), keytable.end(), prop); + if (itr != keytable.end()) { + props_by_index.emplace_back(std::distance(keytable.begin(), itr)); + } + } + + bool needAllProperties = property_filter_type == Filters::filter_properties_types::all; + layer.for_each_feature([&](vtzero::feature&& feature) { mbgl::FeatureType geometry_type = convertGeom(feature.geometry_type()); @@ -348,7 +364,16 @@ void filterFeatures(vtzero::tile_builder* finalvt, feature_builder.set_id(feature.id()); } feature_builder.set_geometry(feature.geometry()); + while (auto idxs = feature.next_property_indexes()) { + if (!needAllProperties) { + // get the key only if we don't need all the properties; + // if the key is not in the properties list, skip to add to feature + if (std::find(props_by_index.begin(), props_by_index.end(), idxs.key()) == props_by_index.end()) { + continue; + } + } + // only if we want all the properties or the key in the properties list we add this property to feature feature_builder.add_property(mapper(idxs)); } feature_builder.commit(); @@ -392,8 +417,9 @@ void AsyncShave(uv_work_t* req) { // get info from tuple auto const& mbgl_filter_obj = std::get<0>(filter); - auto const minzoom = std::get<1>(filter); - auto const maxzoom = std::get<2>(filter); + auto const& property_filter = std::get<1>(filter); + auto const minzoom = std::get<2>(filter); + auto const maxzoom = std::get<3>(filter); // If zoom level is relevant to filter // OR if the style layer minzoom is styling overzoomed tiles... @@ -401,12 +427,12 @@ void AsyncShave(uv_work_t* req) { if ((baton->zoom >= minzoom && baton->zoom <= maxzoom) || (baton->maxzoom && *(baton->maxzoom) < minzoom)) { - // Skip feature re-encoding when filter is null/empty - if (std::get<0>(filter) == mbgl::style::Filter()) { + // Skip feature re-encoding when filter is null/empty AND we have no property k/v filter + if (std::get<0>(filter) == mbgl::style::Filter() && property_filter.first == Filters::filter_properties_types::all) { finalvt.add_existing_layer(layer); // Add to new tile } else { // Ampersand in front of var: "Pass as pointers" - filterFeatures(&finalvt, baton->zoom, layer, mbgl_filter_obj); + filterFeatures(&finalvt, baton->zoom, layer, mbgl_filter_obj, property_filter); } } } diff --git a/test/fixtures/filters/bright-filter.json b/test/fixtures/filters/bright-filter.json index 2c9d55f..4eec784 100644 --- a/test/fixtures/filters/bright-filter.json +++ b/test/fixtures/filters/bright-filter.json @@ -1 +1 @@ -{"landuse_overlay":{"filters":["any",["==","class","national_park"]],"minzoom":0,"maxzoom":22},"landuse":{"filters":["any",["==","class","park"],["==","class","school"],["==","class","wood"]],"minzoom":0,"maxzoom":22},"waterway":{"filters":["any",["all",["!=","class","river"],["!=","class","stream"],["!=","class","canal"]],["==","class","river"],["in","class","stream","canal"]],"minzoom":0,"maxzoom":22},"water":{"filters":true,"minzoom":0,"maxzoom":22},"aeroway":{"filters":["any",["==","$type","Polygon"],["all",["==","$type","LineString"],["==","type","runway"]],["all",["==","$type","LineString"],["==","type","taxiway"]]],"minzoom":11,"maxzoom":22},"road":{"filters":["any",["all",["==","structure","tunnel"],["==","class","motorway_link"]],["all",["==","structure","tunnel"],["in","class","secondary","tertiary"]]],"minzoom":0,"maxzoom":22}} \ No newline at end of file +{"landuse_overlay":{"filters":["any",["==","class","national_park"]],"minzoom":0,"maxzoom":22,"properties":["class"]},"landuse":{"filters":["any",["==","class","park"],["==","class","school"],["==","class","wood"]],"minzoom":0,"maxzoom":22,"properties":["class"]},"waterway":{"filters":["any",["all",["!=","class","river"],["!=","class","stream"],["!=","class","canal"]],["==","class","river"],["in","class","stream","canal"]],"minzoom":0,"maxzoom":22,"properties":["class"]},"water":{"filters":true,"minzoom":0,"maxzoom":22,"properties":[]},"aeroway":{"filters":["any",["==","$type","Polygon"],["all",["==","$type","LineString"],["==","type","runway"]],["all",["==","$type","LineString"],["==","type","taxiway"]]],"minzoom":11,"maxzoom":22,"properties":["type"]},"road":{"filters":["any",["all",["==","structure","tunnel"],["==","class","motorway_link"]],["all",["==","structure","tunnel"],["in","class","secondary","tertiary"]]],"minzoom":0,"maxzoom":22,"properties":["structure","class"]}} \ No newline at end of file diff --git a/test/fixtures/filters/expressions-filter.json b/test/fixtures/filters/expressions-filter.json index 4e27b41..2dde9b4 100644 --- a/test/fixtures/filters/expressions-filter.json +++ b/test/fixtures/filters/expressions-filter.json @@ -1 +1 @@ -{"landcover":{"filters":["any",["step",["zoom"],true,7,["==","class","snow"]]],"minzoom":0,"maxzoom":22},"road":{"filters":["any",["all",["==",["geometry-type"],"LineString"],["match",["get","structure"],"tunnel",true,false],["step",["zoom"],["any",["match",["get","class"],["street","street_limited","track"],true,false],["match",["get","type"],"primary_link",true,false]],14,["any",["match",["get","class"],["street","street_limited","track","service"],true,false],["match",["get","type"],["primary_link","secondary_link","tertiary_link"],true,false]]]],["all",["==","$type","LineString"],["all",["in","class","primary","secondary","tertiary"],["==","structure","tunnel"]]]],"minzoom":0,"maxzoom":22},"building":{"filters":["any",["all",["!=","type","building:part"],["==","underground","false"]]],"minzoom":15,"maxzoom":22}} \ No newline at end of file +{"landcover":{"filters":["any",["step",["zoom"],true,7,["==","class","snow"]]],"minzoom":0,"maxzoom":22,"properties":["class"]},"road":{"filters":["any",["all",["==",["geometry-type"],"LineString"],["match",["get","structure"],"tunnel",true,false],["step",["zoom"],["any",["match",["get","class"],["street","street_limited","track"],true,false],["match",["get","type"],"primary_link",true,false]],14,["any",["match",["get","class"],["street","street_limited","track","service"],true,false],["match",["get","type"],["primary_link","secondary_link","tertiary_link"],true,false]]]],["all",["==","$type","LineString"],["all",["in","class","primary","secondary","tertiary"],["==","structure","tunnel"]]]],"minzoom":0,"maxzoom":22,"properties":["structure","class","type","street_limited","secondary_link"]},"building":{"filters":["any",["all",["!=","type","building:part"],["==","underground","false"]]],"minzoom":15,"maxzoom":22,"properties":["type","underground"]}} \ No newline at end of file diff --git a/test/fixtures/filters/expressions-properties.json b/test/fixtures/filters/expressions-properties.json new file mode 100644 index 0000000..1c553dc --- /dev/null +++ b/test/fixtures/filters/expressions-properties.json @@ -0,0 +1 @@ +{"landuse":{"filters":true,"minzoom":0,"maxzoom":22,"properties":["p1","p2","p3","p4","p5"]},"water":{"filters":true,"minzoom":0,"maxzoom":22,"properties":true}} \ No newline at end of file diff --git a/test/fixtures/filters/floating-filter.json b/test/fixtures/filters/floating-filter.json index b90eee4..036ef70 100644 --- a/test/fixtures/filters/floating-filter.json +++ b/test/fixtures/filters/floating-filter.json @@ -1 +1 @@ -{"landcover":{"filters":true,"minzoom":10.9999999999999,"maxzoom":11.0000000000001}} \ No newline at end of file +{"landcover":{"filters":true,"minzoom":10.9999999999999,"maxzoom":11.0000000000001,"properties":[]}} \ No newline at end of file diff --git a/test/fixtures/properties/floating-filter.json b/test/fixtures/properties/floating-filter.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/test/fixtures/properties/floating-filter.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/fixtures/styles/expressions.json b/test/fixtures/styles/expressions.json index 829fcf9..cbe5ebe 100644 --- a/test/fixtures/styles/expressions.json +++ b/test/fixtures/styles/expressions.json @@ -1,19 +1,33 @@ { "version": 1, "name": "Expressions", - "layers": [ - { + "layers": [{ "id": "landuse", "type": "fill", "source": "composite", "source-layer": "landuse", - "filter": ["match", ["get", "class"], ["airport", "cemetery", "hospital", "park", "pitch", "sand", "school"], true, false] + "filter": ["match", ["get", "class"], + ["airport", "cemetery", "hospital", "park", "pitch", "sand", "school"], true, false + ], + "paint": { + "line-cap": "round", + "expression-test4": ["feature-state", "underground4"], + "expression-test3": ["==", ["feature-state", "underground"], "false"], + "expression-test2": ["==", ["has", "underground"], "false"], + "expression-test": ["==", ["get", "underground1"], "false"], + "expression-test2-fake": ["==", ["has", "underground1", { "obj": 1 }], "false"], + "expression-test5": ["==", ["get", "class"], "false"] + } }, - { + { "id": "water-shadow", "type": "fill", "source": "composite", - "source-layer": "water" + "source-layer": "water", + "layout": { + "icon-image": "{maki}-{what}-{ever}", + "expression-test4": ["==", ["properties"], "false"] + } }, { "id": "building", @@ -21,8 +35,7 @@ "source": "composite", "source-layer": "building", "minzoom": 15, - "filter": ["all", - ["!", ["match", ["get", "type"], "building:part", true, false]], + "filter": ["all", ["!", ["match", ["get", "type"], "building:part", true, false]], ["==", ["get", "underground"], "false"] ] }, @@ -32,18 +45,30 @@ "source": "composite", "source-layer": "road", "minzoom": 12, - "filter": ["all", - ["==", ["geometry-type"], "Polygon"], - ["match", ["get", "structure"], ["none", "ford"], true, false], - ["match", ["get", "class"], ["path", "pedestrian"], true, false] - ] + "filter": ["all", ["==", ["geometry-type"], "Polygon"], + ["match", ["get", "structure"], + ["none", "ford"], true, false + ], + ["match", ["get", "class"], + ["path", "pedestrian"], true, false + ] + ], + "paint": { + "whatervey": "{oneway}" + } }, { "id": "poi-label", "type": "symbol", "source": "composite", "source-layer": "poi_label", - "filter": ["<=", ["number", ["get", "filterrank"]], 3] + "filter": ["<=", ["number", ["get", "filterrank"]], 3], + "paint": { + "circle-radius": "{circle-radius}" + }, + "layout": { + "text-field": "{name_zh}" + } }, { "id": "water", "ref": "water-shadow", "paint": { "fill-color": "hsl(196, 80%, 70%)" } }, { @@ -51,7 +76,15 @@ "type": "symbol", "source": "composite", "source-layer": "housenum_label", - "minzoom": 17 + "minzoom": 17, + "layout": { + "text-field": { + "stops": [ + [11, "{ref}"], + [12, "{name_zh}"] + ] + } + } } ] } \ No newline at end of file diff --git a/test/fixtures/styles/properties.json b/test/fixtures/styles/properties.json new file mode 100644 index 0000000..169231b --- /dev/null +++ b/test/fixtures/styles/properties.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "name": "Expressions", + "layers": [{ + "id": "landuse", + "type": "fill", + "source-layer": "landuse", + "layout": { + "text-field": "{class}" + } + }, + { + "id": "water", + "type": "fill", + "source-layer": "water" + }, + { + "id": "building", + "type": "fill", + "source-layer": "building" + }, + { + "id": "road", + "type": "fill", + "source-layer": "road", + "layout": { + "text-field": "{type}" + }, + "paint": { + "expression-test-fake1": ["oneway"], + "expression-test-fake2": ["==", ["has", "structure", { "obj": 1 }], "false"], + "expression-test": ["==", ["has", "type"], "false"] + } + }, + { + "id": "poi_label", + "type": "fill", + "source-layer": "poi_label", + "layout": { + "expression-all": ["==", ["properties"], "false"] + } + }, + { + "id": "road_label", + "type": "fill", + "source-layer": "road_label" + }, + { + "id": "housenum_label", + "type": "fill", + "source-layer": "housenum_label" + } + ] +} \ No newline at end of file diff --git a/test/mvtfixtures.test.js b/test/mvtfixtures.test.js index 83352c9..11c5fb1 100644 --- a/test/mvtfixtures.test.js +++ b/test/mvtfixtures.test.js @@ -7,28 +7,28 @@ var fixtures = require('@mapbox/mvt-fixtures'); var SHOW_ERROR = process.env.SHOW_ERROR; var genericFilter = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "layer_name", - filter: ["==","string","hello"] - } - ] + layers: [{ + "source-layer": "layer_name", + filter: ["==", "string", "hello"] + }] })); test('validator: layers successfully shaved, all value types', function(t) { var buffer = fixtures.get('038').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==","string_value","ello"] + layers: [{ + "source-layer": "hello", + "filter": ["==", "string_value", "ello"], + "layout": { + "allproperties": ["==", ["properties"], "false"] } - ] + }], + })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { if (err) throw err; - var postTile = new vt(new pbf(shavedTile)); + var postTile = new vt(new pbf(shavedTile)); t.ok(shavedTile); t.equals(Object.keys(postTile.layers).length, 1, 'shaved tile contains expected number of layers'); t.equals(shavedTile.length, 176, 'expected tile size after filtering'); @@ -39,17 +39,18 @@ test('validator: layers successfully shaved, all value types', function(t) { test('validator: layers successfully shaved, expression', function(t) { var buffer = fixtures.get('038').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==", ["get", "string_value"], "ello"] + layers: [{ + "source-layer": "hello", + "filter": ["==", ["get", "string_value"], "ello"], + "layout": { + "allproperties": ["==", ["properties"], "false"] } - ] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { if (err) throw err; - var postTile = new vt(new pbf(shavedTile)); + var postTile = new vt(new pbf(shavedTile)); t.ok(shavedTile); t.equals(Object.keys(postTile.layers).length, 1, 'shaved tile contains expected number of layers'); t.equals(shavedTile.length, 176, 'expected tile size after filtering'); @@ -60,17 +61,18 @@ test('validator: layers successfully shaved, expression', function(t) { test('validator: layers successfully shaved, expression - getType', function(t) { var buffer = fixtures.get('021').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==", ["geometry-type"], "LineString"] + layers: [{ + "source-layer": "hello", + "filter": ["==", ["geometry-type"], "LineString"], + "layout": { + "allproperties": ["==", ["properties"], "false"] } - ] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { if (err) throw err; - var postTile = new vt(new pbf(shavedTile)); + var postTile = new vt(new pbf(shavedTile)); t.ok(shavedTile); t.equals(Object.keys(postTile.layers).length, 1, 'shaved tile contains expected number of layers'); t.equals(shavedTile.length, 56, 'expected tile size after filtering'); @@ -80,7 +82,7 @@ test('validator: layers successfully shaved, expression - getType', function(t) test('validator: version 2 no name field in Layer', function(t) { var buffer = fixtures.get('014').buffer; - Shaver.shave(buffer, {filters: genericFilter, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: genericFilter, zoom: 0 }, function(err, shavedTile) { t.ok(err); if (SHOW_ERROR) console.log(err); t.end(); @@ -91,15 +93,13 @@ test('validator: unknown field type in Layer', function(t) { var buffer = fixtures.get('016').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==","id","1"] - } - ] + layers: [{ + "source-layer": "hello", + filter: ["==", "id", "1"] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { t.notOk(err); t.end(); }); @@ -108,7 +108,7 @@ test('validator: unknown field type in Layer', function(t) { test('validator: version 1 no name', function(t) { var buffer = fixtures.get('023').buffer; - Shaver.shave(buffer, {filters: genericFilter, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: genericFilter, zoom: 0 }, function(err, shavedTile) { t.ok(err); if (SHOW_ERROR) console.log(err); t.end(); @@ -119,15 +119,13 @@ test('validator: odd number of tags in Feature', function(t) { var buffer = fixtures.get('005').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==","string_value","world"] - } - ] + layers: [{ + "source-layer": "hello", + filter: ["==", "string_value", "world"] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { t.ok(err); if (SHOW_ERROR) console.log(err); t.end(); @@ -138,15 +136,13 @@ test('validator: invalid key or value as it does not appear in the layer', funct var buffer = fixtures.get('042').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==","string_value","park"] - } - ] + layers: [{ + "source-layer": "hello", + filter: ["==", "string_value", "park"] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { t.ok(err); if (SHOW_ERROR) console.log(err); t.end(); @@ -156,16 +152,14 @@ test('validator: invalid key or value as it does not appear in the layer', funct test('validator: Feature unknown geometry type', function(t) { var buffer = fixtures.get('006').buffer; - var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - "filter": [ "==", "$id", 0 ] - } - ] + var filters = new Shaver.Filters(Shaver.styleToFilters({ + layers: [{ + "source-layer": "hello", + "filter": ["==", "$id", 0] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { t.ok(err); if (SHOW_ERROR) console.log(err); t.end(); @@ -176,15 +170,13 @@ test('validator: Feature unknown field type type', function(t) { var buffer = fixtures.get('041').buffer; var filters = new Shaver.Filters(Shaver.styleToFilters({ - layers: [ - { - "source-layer": "hello", - filter: ["==","string_value","lake"] - } - ] + layers: [{ + "source-layer": "hello", + filter: ["==", "string_value", "lake"] + }] })); - Shaver.shave(buffer, {filters: filters, zoom: 0}, function(err, shavedTile) { + Shaver.shave(buffer, { filters: filters, zoom: 0 }, function(err, shavedTile) { t.ok(err); if (SHOW_ERROR) console.log(err); t.end(); diff --git a/test/propertyKeyValueFilter-Error.test.js b/test/propertyKeyValueFilter-Error.test.js new file mode 100644 index 0000000..294133d --- /dev/null +++ b/test/propertyKeyValueFilter-Error.test.js @@ -0,0 +1,58 @@ +var Shaver = require('../'); +var fs = require('fs'); +var vt = require('@mapbox/vector-tile').VectorTile; +var pbf = require('pbf'); +var test = require('tape'); +var path = require('path'); +var propertyrJSON = './fixtures/properties/floating-filter.json'; + + +var sfTileBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/sf_16_10465_25329.vector.pbf'); +var z16HousenumBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/z16-housenum.mvt'); +var filter_obj = Shaver.styleToFilters(JSON.parse(fs.readFileSync('./test/fixtures/styles/properties.json').toString())); + + + + +test('property key value filter size check', t => { + let FilterObj = Shaver.styleToFilters({ + "layers": [ + { "id": "landuse", "source-layer": "landuse" } + ] + }); + FilterObj.landuse.properties = null; + // var filters = new Shaver.Filters(null); + // FilterObj.landuse.properties = null + + try { + var filters = new Shaver.Filters(FilterObj, (err) => { + console.log(err); + }); + } catch (err) { + t.ok(err); + t.equal(err.message, 'Property-Filters is not properly constructed.', 'expected error message'); + t.end(); + } +}); + + +test('property key value filter size check', t => { + let FilterObj = Shaver.styleToFilters({ + "layers": [{ + "id": "landuse", + "source-layer": "landuse", + layout: { "expression-test4": ["==", ["properties"], "false"] } + }] + }); + + try { + FilterObj.landuse.properties = false; + var filters = new Shaver.Filters(FilterObj, (err) => { + console.log(err); + }); + } catch (err) { + t.ok(err); + t.equal(err.message, 'invalid filter value, must be an array or a boolean', 'expected error message'); + t.end(); + } +}); \ No newline at end of file diff --git a/test/propertyKeyValueFilter.test.js b/test/propertyKeyValueFilter.test.js new file mode 100644 index 0000000..a528198 --- /dev/null +++ b/test/propertyKeyValueFilter.test.js @@ -0,0 +1,80 @@ +var Shaver = require('../'); +var fs = require('fs'); +var vt = require('@mapbox/vector-tile').VectorTile; +var pbf = require('pbf'); +var test = require('tape'); +var path = require('path'); +var propertyrJSON = './fixtures/properties/floating-filter.json'; + + +var sfTileBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/sf_16_10465_25329.vector.pbf'); +var z16HousenumBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/z16-housenum.mvt'); +var filter_obj = Shaver.styleToFilters(JSON.parse(fs.readFileSync('./test/fixtures/styles/properties.json').toString())); + + + +// test expressin +function vtinfo(buffer) { + var tile = new vt(new pbf(buffer)); + var layerInfo = {}; + var info = { + layers: [] + }; + Object.keys(tile.layers).forEach(function(k) { + var lay = tile.layers[k]; + let propertyKies = {}; + for (var i = 0; i < lay.length; i++) { + let features = lay.feature(i).toGeoJSON(0, 0, 0); + Object.keys(features.properties).forEach(key => { + propertyKies[key] = true; + }); + } + + layerInfo[k] = { + features: lay.length, + properties: JSON.stringify(Object.keys(propertyKies)) + } + }); + return layerInfo; +} + + +test('property key value filter size check', t => { + var filters = new Shaver.Filters(Shaver.styleToFilters({ + "layers": [ + { "id": "landuse", "source-layer": "landuse" }, + { "id": "water", "source-layer": "water" }, + { "id": "building", "source-layer": "building" }, + { "id": "road", "source-layer": "road" }, + { "id": "poi_label", "source-layer": "poi_label" }, + { "id": "road_label", "source-layer": "road_label" }, + { "id": "housenum_label", "source-layer": "housenum_label" } + ] + })); + Shaver.shave(sfTileBuffer, { filters, zoom: 14 }, function(err, shavedTile) { + if (err) throw err; + t.equals(sfTileBuffer.length, 7718, 'the size before shave of sf tile'); + t.equals(shavedTile.length, 5514, 'the size after the shave of sf tile'); + }); + Shaver.shave(z16HousenumBuffer, { filters, zoom: 14 }, function(err, shavedTile) { + if (err) throw err; + t.equals(z16HousenumBuffer.length, 30607, 'the size before shave of z16 Housenum'); + t.equals(shavedTile.length, 16780, 'the size after the shave of z16 Housenum'); + }); + t.end(); +}); + + +test('property key value filter', t => { + var filters = new Shaver.Filters(filter_obj); + Shaver.shave(sfTileBuffer, { filters, zoom: 14 }, function(err, shavedTile) { + if (err) throw err; + t.equals(sfTileBuffer.length, 7718, 'the size before shave in round2 test'); + t.equals(shavedTile.length, 6609, 'the size after the shave in round2 test'); + if (process.env.UPDATE) { + fs.writeFileSync(path.resolve(__dirname, propertyrJSON), JSON.stringify(filters)); + } + t.deepEquals(filters, require(propertyrJSON), 'property key value filter correctly'); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/speed.js b/test/speed.js new file mode 100644 index 0000000..9016989 --- /dev/null +++ b/test/speed.js @@ -0,0 +1,32 @@ +var Shaver = require('../'); +var fs = require('fs'); +var vt = require('@mapbox/vector-tile').VectorTile; +var pbf = require('pbf'); +var test = require('tape'); +var path = require('path'); +var propertyrJSON = './fixtures/properties/floating-filter.json'; + + +var sfTileBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/sf_16_10465_25329.vector.pbf'); +var z16HousenumBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/z16-housenum.mvt'); +// var filter_obj = Shaver.styleToFilters(JSON.parse(fs.readFileSync('./test/fixtures/styles/properties.json').toString())); + + + +var filter_obj = Shaver.styleToFilters({ + "layers": [ + { "id": "landuse", "source-layer": "landuse", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } }, + { "id": "water", "source-layer": "water", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } }, + { "id": "building", "source-layer": "building", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } }, + { "id": "road", "source-layer": "road", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } }, + { "id": "poi_label", "source-layer": "poi_label", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } }, + { "id": "road_label", "source-layer": "road_label", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } }, + { "id": "housenum_label", "source-layer": "housenum_label", filter: true, layout: { "expression-test5": ["==", ["get", "class"], "false"] } } + ] +}); +var filters = new Shaver.Filters(filter_obj); +for (var i = 0; i < 1000000; i++) { + Shaver.shave(sfTileBuffer, { filters, zoom: 14 }, function(err, shavedTile) {}); +} + +console.log('done') \ No newline at end of file diff --git a/test/styleToFilter-property.test.js b/test/styleToFilter-property.test.js new file mode 100644 index 0000000..d4b49d1 --- /dev/null +++ b/test/styleToFilter-property.test.js @@ -0,0 +1,43 @@ +var fs = require('fs'); +var path = require('path'); +var test = require('tape'); +var styleToFilter = require('../lib/styleToFilters.js'); +var properties_result_expressions = './fixtures/filters/expressions-properties.json'; + +test('test get used properites from style.json', function(t) { + var filters = styleToFilter({ + "layers": [{ + "source-layer": "landuse", + "paint": { + "exp-test1": ["==", ["get", "p1"], "false"], + "exp-test1-fake": ["==", ["get", "p1-fake", { "obj": 1 }], "false"], + "exp-test2": ["==", ["has", "p2"], "false"], + "exp-test2-fake": ["==", ["has", "p2-fake", { "obj": 1 }], "false"], + "exp-test3": ["==", ["feature-state", "p3"], "false"], + "exp-test4": ["feature-state", "p4"], + "exp-test5": { + "property": "p5" + }, + } + }, { + "source-layer": "water", + "paint": { + "exp-test0": ["properties"], + "exp-test1": ["==", ["get", "p1"], "false"], + "exp-test1-fake": ["==", ["get", "p1-fake", { "obj": 1 }], "false"], + "exp-test2": ["==", ["has", "p2"], "false"], + "exp-test2-fake": ["==", ["has", "p2-fake", { "obj": 1 }], "false"], + "exp-test3": ["==", ["feature-state", "p3"], "false"], + "exp-test4": ["feature-state", "p4"], + } + }] + }); + // console.log('xxx', filters); + if (process.env.UPDATE) { + console.log('> UPDATING ' + properties_result_expressions); + fs.writeFileSync(path.resolve(__dirname, properties_result_expressions), JSON.stringify(filters)); + } + t.deepEquals(filters, require(properties_result_expressions), 'expressions filter is extracted correctly'); + + t.end(); +}); \ No newline at end of file diff --git a/test/styleToFilter.test.js b/test/styleToFilter.test.js index 581a1c4..1c0cec8 100644 --- a/test/styleToFilter.test.js +++ b/test/styleToFilter.test.js @@ -14,113 +14,128 @@ test('error handling', function(t) { t.deepEqual(styleToFilter({}), {}, 'returns a plain object when given a plain object'); t.deepEqual(styleToFilter([]), {}, 'returns a plain object when given an array'); t.deepEqual(styleToFilter('hello'), {}, 'returns a plain object when given a string'); - t.deepEqual(styleToFilter({layers:[]}), {}, 'returns a plain object when given an empty style layers'); - t.deepEqual(styleToFilter({layers:'lol no layers here'}), {}, 'returns a plain object when given snarky style layers'); - + t.deepEqual(styleToFilter({ layers: [] }), {}, 'returns a plain object when given an empty style layers'); + t.deepEqual(styleToFilter({ layers: 'lol no layers here' }), {}, 'returns a plain object when given snarky style layers'); t.end(); }); test('min/max zoom defaults are set if the do not exist', function(t) { t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water' - } - ] - }), {water: {filters: true, minzoom: 0, maxzoom: 22}}, 'returns water:true for only water layer and includes min/max zoom'); + layers: [{ + 'source-layer': 'water' + }] + }), { water: { filters: true, minzoom: 0, maxzoom: 22, properties: [] } }, 'returns water:true for only water layer and includes min/max zoom'); t.end(); }); test('simple style layers', function(t) { - t.deepEqual(styleToFilter({layers:[{arbitrary:'layer'}]}), {}, 'skips any layers without source-layer key'); + t.deepEqual(styleToFilter({ layers: [{ arbitrary: 'layer' }] }), {}, 'skips any layers without source-layer key'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water', - minzoom: 10, - maxzoom: 15 - } - ] - }), {water: {filters: true, minzoom: 10, maxzoom: 15}}, 'returns water:true for only water layer and includes min/max zoom'); + layers: [{ + 'source-layer': 'water', + minzoom: 10, + maxzoom: 15 + }] + }), { water: { filters: true, minzoom: 10, maxzoom: 15, properties: [] } }, 'returns water:true for only water layer and includes min/max zoom'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water', - filter: ['==','color','blue'] - } - ] - }), {water: {filters: ['any',['==','color','blue']], minzoom: 0, maxzoom: 22}}, 'returns water:filter for water layer with filter'); + layers: [{ + 'source-layer': 'water', + filter: ['==', 'color', 'blue'] + }] + }), { water: { filters: ['any', ['==', 'color', 'blue']], minzoom: 0, maxzoom: 22, properties: ['color'] } }, 'returns water:filter for water layer with filter'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water' + layers: [{ + 'source-layer': 'water' }, { - 'source-layer':'water', - filter: ['==','color','blue'] + 'source-layer': 'water', + filter: ['==', 'color', 'blue'] } ] - }), {water:{filters: true, minzoom: 0, maxzoom: 22}}, 'returns water:filter for multiple water layers, some with filters'); + }), { water: { filters: true, minzoom: 0, maxzoom: 22, properties: ['color'] } }, 'returns water:filter for multiple water layers, some with filters'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water', - filter: ['!=','color','blue'], + layers: [{ + 'source-layer': 'water', + filter: ['!=', 'color', 'blue'], minzoom: 10, maxzoom: 15 }, { - 'source-layer':'water', - filter: ['==','color','blue'], + 'source-layer': 'water', + filter: ['==', 'color', 'blue'], minzoom: 8, maxzoom: 16 } ] - }), {water: {filters: ['any',['!=','color','blue'],['==','color','blue']], minzoom: 8, maxzoom: 16}}, 'returns water:filter for multiple water filters, and updates min/max zoom for smallest/largest values'); + }), { + water: { + filters: ['any', ['!=', 'color', 'blue'], + ['==', 'color', 'blue'] + ], + minzoom: 8, + maxzoom: 16, + properties: ['color'] + } + }, 'returns water:filter for multiple water filters, and updates min/max zoom for smallest/largest values'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water', - filter: ['!=','color','blue'], + layers: [{ + 'source-layer': 'water', + filter: ['!=', 'color', 'blue'], minzoom: 10, maxzoom: 15 }, { - 'source-layer':'water', - filter: ['==','color','blue'] + 'source-layer': 'water', + filter: ['==', 'color', 'blue'] } ] - }), {water: {filters: ['any',['!=','color','blue'],['==','color','blue']], minzoom: 0, maxzoom: 22}}, 'returns water:filter for multiple water filters, and updates min/max zoom to 0 and 22 if one filter doesn\'t have zooms'); + }), { + water: { + filters: ['any', ['!=', 'color', 'blue'], + ['==', 'color', 'blue'] + ], + minzoom: 0, + maxzoom: 22, + properties: ['color'] + } + }, 'returns water:filter for multiple water filters, and updates min/max zoom to 0 and 22 if one filter doesn\'t have zooms'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water', - filter: ['!=','color','blue'] + layers: [{ + 'source-layer': 'water', + filter: ['!=', 'color', 'blue'] }, { - 'source-layer':'water', - filter: ['==','color','blue'] + 'source-layer': 'water', + filter: ['==', 'color', 'blue'] } ] - }), {water: {filters: ['any',['!=','color','blue'],['==','color','blue']], minzoom: 0, maxzoom: 22}}, 'returns water:filter for multiple water layers with filters'); + }), { + water: { + filters: ['any', ['!=', 'color', 'blue'], + ['==', 'color', 'blue'] + ], + minzoom: 0, + maxzoom: 22, + properties: ['color'] + } + }, 'returns water:filter for multiple water layers with filters'); t.deepEqual(styleToFilter({ - layers: [ - { - 'source-layer':'water', - filter: ['!=','color','blue'] + layers: [{ + 'source-layer': 'water', + filter: ['!=', 'color', 'blue'] }, { - 'source-layer':'landcover', - filter: ['==','color','blue'] + 'source-layer': 'landcover', + filter: ['==', 'color', 'blue'] } ] - }), {water: {filters: ['any',['!=','color','blue']], minzoom: 0, maxzoom: 22},landcover: {filters: ['any',['==','color','blue']], minzoom: 0, maxzoom: 22}}, 'returns right filters for multiple layers with filters'); + }), { water: { filters: ['any', ['!=', 'color', 'blue']], minzoom: 0, maxzoom: 22, properties: ['color'] }, landcover: { filters: ['any', ['==', 'color', 'blue']], minzoom: 0, maxzoom: 22, properties: ['color'] } }, 'returns right filters for multiple layers with filters'); t.end(); }); diff --git a/test/temp.js b/test/temp.js new file mode 100644 index 0000000..a528198 --- /dev/null +++ b/test/temp.js @@ -0,0 +1,80 @@ +var Shaver = require('../'); +var fs = require('fs'); +var vt = require('@mapbox/vector-tile').VectorTile; +var pbf = require('pbf'); +var test = require('tape'); +var path = require('path'); +var propertyrJSON = './fixtures/properties/floating-filter.json'; + + +var sfTileBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/sf_16_10465_25329.vector.pbf'); +var z16HousenumBuffer = fs.readFileSync(__dirname + '/fixtures/tiles/z16-housenum.mvt'); +var filter_obj = Shaver.styleToFilters(JSON.parse(fs.readFileSync('./test/fixtures/styles/properties.json').toString())); + + + +// test expressin +function vtinfo(buffer) { + var tile = new vt(new pbf(buffer)); + var layerInfo = {}; + var info = { + layers: [] + }; + Object.keys(tile.layers).forEach(function(k) { + var lay = tile.layers[k]; + let propertyKies = {}; + for (var i = 0; i < lay.length; i++) { + let features = lay.feature(i).toGeoJSON(0, 0, 0); + Object.keys(features.properties).forEach(key => { + propertyKies[key] = true; + }); + } + + layerInfo[k] = { + features: lay.length, + properties: JSON.stringify(Object.keys(propertyKies)) + } + }); + return layerInfo; +} + + +test('property key value filter size check', t => { + var filters = new Shaver.Filters(Shaver.styleToFilters({ + "layers": [ + { "id": "landuse", "source-layer": "landuse" }, + { "id": "water", "source-layer": "water" }, + { "id": "building", "source-layer": "building" }, + { "id": "road", "source-layer": "road" }, + { "id": "poi_label", "source-layer": "poi_label" }, + { "id": "road_label", "source-layer": "road_label" }, + { "id": "housenum_label", "source-layer": "housenum_label" } + ] + })); + Shaver.shave(sfTileBuffer, { filters, zoom: 14 }, function(err, shavedTile) { + if (err) throw err; + t.equals(sfTileBuffer.length, 7718, 'the size before shave of sf tile'); + t.equals(shavedTile.length, 5514, 'the size after the shave of sf tile'); + }); + Shaver.shave(z16HousenumBuffer, { filters, zoom: 14 }, function(err, shavedTile) { + if (err) throw err; + t.equals(z16HousenumBuffer.length, 30607, 'the size before shave of z16 Housenum'); + t.equals(shavedTile.length, 16780, 'the size after the shave of z16 Housenum'); + }); + t.end(); +}); + + +test('property key value filter', t => { + var filters = new Shaver.Filters(filter_obj); + Shaver.shave(sfTileBuffer, { filters, zoom: 14 }, function(err, shavedTile) { + if (err) throw err; + t.equals(sfTileBuffer.length, 7718, 'the size before shave in round2 test'); + t.equals(shavedTile.length, 6609, 'the size after the shave in round2 test'); + if (process.env.UPDATE) { + fs.writeFileSync(path.resolve(__dirname, propertyrJSON), JSON.stringify(filters)); + } + t.deepEquals(filters, require(propertyrJSON), 'property key value filter correctly'); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/vtshaver.test.js b/test/vtshaver.test.js index e76e950..179a431 100644 --- a/test/vtshaver.test.js +++ b/test/vtshaver.test.js @@ -702,7 +702,7 @@ test('failure: creating Shaver.Filters() with invalid object', function(t) { test('failure: Shaver.Filters(): invalid filter', function(t) { try { var filters = new Shaver.Filters({ - "poi_label": {filters: 2, minzoom: 0, maxzoom: 22 } + "poi_label": {filters: 2, minzoom: 0, maxzoom: 22, properties: true } }); t.ok(false); } catch (err) { @@ -714,7 +714,7 @@ test('failure: Shaver.Filters(): invalid filter', function(t) { test('failure: creating Shaver.Filters() with invalid Filter object', function(t) { try { - var filters = new Shaver.Filters({ "poi_label": {filters: [0], minzoom: 0, maxzoom: 22} }); + var filters = new Shaver.Filters({ "poi_label": {filters: [0], minzoom: 0, maxzoom: 22, properties:true} }); t.ok(false); } catch (err) { t.ok(err);