From a9048e745249683310e8198f0fecd2e39e299d95 Mon Sep 17 00:00:00 2001 From: Jim Unger Date: Tue, 26 Apr 2016 13:54:54 -0500 Subject: [PATCH 1/3] [add data] hides disabled processors --- .../directives/pipeline_setup.js | 12 ++- .../process_es_ingest_processors_response.js | 90 +++++++++++++++++++ .../process_es_ingest_processors_response.js | 16 ++++ .../kibana/server/routes/api/ingest/index.js | 2 + .../routes/api/ingest/register_processors.js | 24 +++++ src/ui/public/ingest/ingest.js | 12 +++ test/unit/api/ingest/_processors.js | 19 ++++ test/unit/api/ingest/index.js | 4 +- 8 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js create mode 100644 src/plugins/kibana/server/lib/process_es_ingest_processors_response.js create mode 100644 src/plugins/kibana/server/routes/api/ingest/register_processors.js create mode 100644 test/unit/api/ingest/_processors.js diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js index c697114dc38167..f6d87bb2d3ff74 100644 --- a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js @@ -11,7 +11,8 @@ import './processor_ui'; import pipelineSetupTemplate from '../views/pipeline_setup.html'; const app = uiModules.get('kibana'); -function buildProcessorTypeList() { + +function buildProcessorTypeList(enabledProcessorTypeIds) { return _(ProcessorTypes) .map(Type => { const instance = new Type(); @@ -22,6 +23,8 @@ function buildProcessorTypeList() { }; }) .compact() + .filter((processorType) => enabledProcessorTypeIds.includes(processorType.typeId)) + .sortBy('title') .value(); } @@ -36,9 +39,14 @@ app.directive('pipelineSetup', function () { controller: function ($scope, debounce, Private, Notifier) { const ingest = Private(IngestProvider); const notify = new Notifier({ location: `Ingest Pipeline Setup` }); - $scope.processorTypes = _.sortBy(buildProcessorTypeList(), 'title'); $scope.sample = {}; + //determines which processors are available on the cluster + ingest.getProcessors() + .then((enabledProcessorTypeIds) => { + $scope.processorTypes = buildProcessorTypeList(enabledProcessorTypeIds); + }); + const pipeline = new Pipeline(); // Loads pre-existing pipeline which will exist if the user returns from // a later step in the wizard diff --git a/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js b/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js new file mode 100644 index 00000000000000..5d5548aa55cbd1 --- /dev/null +++ b/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js @@ -0,0 +1,90 @@ +import processESIngestProcessorsResponse from '../process_es_ingest_processors_response'; +import expect from 'expect.js'; +import _ from 'lodash'; + +describe('processESIngestSimulateResponse', function () { + + it('should return a list of strings indicating the enabled processors', function () { + const response = { + nodes: { + node_foo: { + ingest: { + processors: [ + { type: 'proc_foo' }, + { type: 'proc_bar' } + ] + } + } + } + }; + + const expected = [ 'proc_foo', 'proc_bar' ]; + const actual = processESIngestProcessorsResponse(response); + + expect(actual).to.eql(expected); + }); + + it('should return a unique list of processors', function () { + const response = { + nodes: { + node_foo: { + ingest: { + processors: [ + { type: 'proc_foo' }, + { type: 'proc_bar' } + ] + } + }, + node_bar: { + ingest: { + processors: [ + { type: 'proc_foo' }, + { type: 'proc_bar' } + ] + } + } + } + }; + + const expected = [ 'proc_foo', 'proc_bar' ]; + const actual = processESIngestProcessorsResponse(response); + + expect(actual).to.eql(expected); + }); + + it('should combine the available processors from all nodes', function () { + const response = { + nodes: { + node_foo: { + ingest: { + processors: [ + { type: 'proc_foo' } + ] + } + }, + node_bar: { + ingest: { + processors: [ + { type: 'proc_bar' } + ] + } + } + } + }; + + const expected = [ 'proc_foo', 'proc_bar' ]; + const actual = processESIngestProcessorsResponse(response); + + expect(actual).to.eql(expected); + }); + + it('should return an empty array for unexpected response', function () { + expect(processESIngestProcessorsResponse({ nodes: {}})).to.eql([]); + expect(processESIngestProcessorsResponse({})).to.eql([]); + expect(processESIngestProcessorsResponse(undefined)).to.eql([]); + expect(processESIngestProcessorsResponse(null)).to.eql([]); + expect(processESIngestProcessorsResponse('')).to.eql([]); + expect(processESIngestProcessorsResponse(1)).to.eql([]); + }); + +}); diff --git a/src/plugins/kibana/server/lib/process_es_ingest_processors_response.js b/src/plugins/kibana/server/lib/process_es_ingest_processors_response.js new file mode 100644 index 00000000000000..720ad472641931 --- /dev/null +++ b/src/plugins/kibana/server/lib/process_es_ingest_processors_response.js @@ -0,0 +1,16 @@ +const _ = require('lodash'); + +export default function processESIngestProcessorsResponse(response) { + const nodes = _.get(response, 'nodes'); + + const results = _.chain(nodes) + .map('ingest.processors') + .reduce((result, processors) => { + return result.concat(processors); + }) + .map('type') + .unique() + .value(); + + return results; +}; diff --git a/src/plugins/kibana/server/routes/api/ingest/index.js b/src/plugins/kibana/server/routes/api/ingest/index.js index 5f9fb39747cf26..c0e5bab73d746a 100644 --- a/src/plugins/kibana/server/routes/api/ingest/index.js +++ b/src/plugins/kibana/server/routes/api/ingest/index.js @@ -1,9 +1,11 @@ import { registerPost } from './register_post'; import { registerDelete } from './register_delete'; +import { registerProcessors } from './register_processors'; import { registerSimulate } from './register_simulate'; export default function (server) { registerPost(server); registerDelete(server); + registerProcessors(server); registerSimulate(server); } diff --git a/src/plugins/kibana/server/routes/api/ingest/register_processors.js b/src/plugins/kibana/server/routes/api/ingest/register_processors.js new file mode 100644 index 00000000000000..399e33ea1a852a --- /dev/null +++ b/src/plugins/kibana/server/routes/api/ingest/register_processors.js @@ -0,0 +1,24 @@ +import _ from 'lodash'; +import handleESError from '../../../lib/handle_es_error'; +import handleResponse from '../../../lib/process_es_ingest_processors_response'; +import { keysToCamelCaseShallow, keysToSnakeCaseShallow } from '../../../../common/lib/case_conversion'; + +export function registerProcessors(server) { + server.route({ + path: '/api/kibana/ingest/processors', + method: 'GET', + handler: function (request, reply) { + const boundCallWithRequest = _.partial(server.plugins.elasticsearch.callWithRequest, request); + + return boundCallWithRequest('transport.request', { + path: '/_nodes/ingest', + method: 'GET' + }) + .then(handleResponse) + .then(reply) + .catch((error) => { + reply(handleESError(error)); + }); + } + }); +}; diff --git a/src/ui/public/ingest/ingest.js b/src/ui/public/ingest/ingest.js index a80d1522f6b695..1c933d69673224 100644 --- a/src/ui/public/ingest/ingest.js +++ b/src/ui/public/ingest/ingest.js @@ -59,4 +59,16 @@ export default function IngestProvider($rootScope, $http, config) { }); }; + this.getProcessors = function () { + function unpack(response) { + return response.data; + } + + return $http.get(`${ingestAPIPrefix}/processors`) + .then(unpack) + .catch(err => { + throw ('Error communicating with Kibana server'); + }); + }; + } diff --git a/test/unit/api/ingest/_processors.js b/test/unit/api/ingest/_processors.js new file mode 100644 index 00000000000000..e6732f1b98242a --- /dev/null +++ b/test/unit/api/ingest/_processors.js @@ -0,0 +1,19 @@ +define(function (require) { + var Promise = require('bluebird'); + var _ = require('intern/dojo/node!lodash'); + var expect = require('intern/dojo/node!expect.js'); + + return function (bdd, scenarioManager, request) { + bdd.describe('processors', () => { + + bdd.it('should return 200 for a successful run', function () { + return request.get('/kibana/ingest/listprocessors') + .expect(200) + .then((response) => { + expect(_.isArray(response.body)).to.be(true); + }); + }); + + }); + }; +}); diff --git a/test/unit/api/ingest/index.js b/test/unit/api/ingest/index.js index a57678872bde7a..02860a13ebdb7b 100644 --- a/test/unit/api/ingest/index.js +++ b/test/unit/api/ingest/index.js @@ -9,7 +9,8 @@ define(function (require) { var post = require('./_post'); var del = require('./_del'); var simulate = require('./_simulate'); - var processors = require('./processors/index'); + var processors = require('./_processors'); + var processorTypes = require('./processors/index'); bdd.describe('ingest API', function () { var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch)); @@ -27,5 +28,6 @@ define(function (require) { del(bdd, scenarioManager, request); simulate(bdd, scenarioManager, request); processors(bdd, scenarioManager, request); + processorTypes(bdd, scenarioManager, request); }); }); From 93ea5edaf1ee11e0112de1edd7442dd62d9653e8 Mon Sep 17 00:00:00 2001 From: Jim Unger Date: Wed, 27 Apr 2016 10:23:20 -0500 Subject: [PATCH 2/3] [add data] added error handling when getting processors, and fixed comparisons in tests --- .../directives/pipeline_setup.js | 3 ++- .../process_es_ingest_processors_response.js | 18 +++++++++--------- src/ui/public/ingest/ingest.js | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js index f6d87bb2d3ff74..7b073eb06efe0c 100644 --- a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/pipeline_setup.js @@ -45,7 +45,8 @@ app.directive('pipelineSetup', function () { ingest.getProcessors() .then((enabledProcessorTypeIds) => { $scope.processorTypes = buildProcessorTypeList(enabledProcessorTypeIds); - }); + }) + .catch(notify.error); const pipeline = new Pipeline(); // Loads pre-existing pipeline which will exist if the user returns from diff --git a/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js b/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js index 5d5548aa55cbd1..888f6d352f9bd5 100644 --- a/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js +++ b/src/plugins/kibana/server/lib/__tests__/process_es_ingest_processors_response.js @@ -21,7 +21,7 @@ describe('processESIngestSimulateResponse', function () { const expected = [ 'proc_foo', 'proc_bar' ]; const actual = processESIngestProcessorsResponse(response); - expect(actual).to.eql(expected); + expect(_.isEqual(actual, expected)).to.be.ok(); }); it('should return a unique list of processors', function () { @@ -49,7 +49,7 @@ describe('processESIngestSimulateResponse', function () { const expected = [ 'proc_foo', 'proc_bar' ]; const actual = processESIngestProcessorsResponse(response); - expect(actual).to.eql(expected); + expect(_.isEqual(actual, expected)).to.be.ok(); }); it('should combine the available processors from all nodes', function () { @@ -75,16 +75,16 @@ describe('processESIngestSimulateResponse', function () { const expected = [ 'proc_foo', 'proc_bar' ]; const actual = processESIngestProcessorsResponse(response); - expect(actual).to.eql(expected); + expect(_.isEqual(actual, expected)).to.be.ok(); }); it('should return an empty array for unexpected response', function () { - expect(processESIngestProcessorsResponse({ nodes: {}})).to.eql([]); - expect(processESIngestProcessorsResponse({})).to.eql([]); - expect(processESIngestProcessorsResponse(undefined)).to.eql([]); - expect(processESIngestProcessorsResponse(null)).to.eql([]); - expect(processESIngestProcessorsResponse('')).to.eql([]); - expect(processESIngestProcessorsResponse(1)).to.eql([]); + expect(_.isEqual(processESIngestProcessorsResponse({ nodes: {}}), [])).to.be.ok(); + expect(_.isEqual(processESIngestProcessorsResponse({}), [])).to.be.ok(); + expect(_.isEqual(processESIngestProcessorsResponse(undefined), [])).to.be.ok(); + expect(_.isEqual(processESIngestProcessorsResponse(null), [])).to.be.ok(); + expect(_.isEqual(processESIngestProcessorsResponse(''), [])).to.be.ok(); + expect(_.isEqual(processESIngestProcessorsResponse(1), [])).to.be.ok(); }); }); diff --git a/src/ui/public/ingest/ingest.js b/src/ui/public/ingest/ingest.js index 1c933d69673224..fd14e12590420e 100644 --- a/src/ui/public/ingest/ingest.js +++ b/src/ui/public/ingest/ingest.js @@ -67,7 +67,7 @@ export default function IngestProvider($rootScope, $http, config) { return $http.get(`${ingestAPIPrefix}/processors`) .then(unpack) .catch(err => { - throw ('Error communicating with Kibana server'); + throw ('Error fetching enabled processors'); }); }; From f0e99c3cdb40fe35ec614e3f49310de8727ce980 Mon Sep 17 00:00:00 2001 From: Jim Unger Date: Wed, 27 Apr 2016 15:33:21 -0500 Subject: [PATCH 3/3] [add data] added tests for ingest.getProcessors --- src/ui/public/ingest/__tests__/ingest.js | 32 ++++++++++++++++++++++++ src/ui/public/ingest/ingest.js | 6 ++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/ui/public/ingest/__tests__/ingest.js b/src/ui/public/ingest/__tests__/ingest.js index 1184fcbf5d3630..16dbb7ebf1c4b5 100644 --- a/src/ui/public/ingest/__tests__/ingest.js +++ b/src/ui/public/ingest/__tests__/ingest.js @@ -113,4 +113,36 @@ describe('Ingest Service', function () { expect($rootScope.$broadcast.calledWith('ingest:updated')).to.be.ok(); }); }); + + describe('getProcessors', () => { + + it('Calls the processors GET endpoint of the ingest API', function () { + $httpBackend + .expectGET('../api/kibana/ingest/processors') + .respond('ok'); + + ingest.getProcessors(); + $httpBackend.flush(); + }); + + it('Throws user-friendly error when there is an error in the request', function (done) { + $httpBackend + .when('GET', '../api/kibana/ingest/processors') + .respond(404); + + ingest.getProcessors() + .then( + () => { + throw new Error('expected an error response'); + }, + (error) => { + expect(error.message).to.be('Error fetching enabled processors'); + done(); + }); + + $httpBackend.flush(); + }); + + }); + }); diff --git a/src/ui/public/ingest/ingest.js b/src/ui/public/ingest/ingest.js index fd14e12590420e..bee1cb564d3af3 100644 --- a/src/ui/public/ingest/ingest.js +++ b/src/ui/public/ingest/ingest.js @@ -2,7 +2,7 @@ import { keysToCamelCaseShallow, keysToSnakeCaseShallow } from '../../../plugins import _ from 'lodash'; import angular from 'angular'; -export default function IngestProvider($rootScope, $http, config) { +export default function IngestProvider($rootScope, $http, config, $q) { const ingestAPIPrefix = '../api/kibana/ingest'; @@ -55,7 +55,7 @@ export default function IngestProvider($rootScope, $http, config) { return $http.post(`${ingestAPIPrefix}/simulate`, pack(pipeline)) .then(unpack) .catch(err => { - throw ('Error communicating with Kibana server'); + return $q.reject(new Error('Error simulating pipeline')); }); }; @@ -67,7 +67,7 @@ export default function IngestProvider($rootScope, $http, config) { return $http.get(`${ingestAPIPrefix}/processors`) .then(unpack) .catch(err => { - throw ('Error fetching enabled processors'); + return $q.reject(new Error('Error fetching enabled processors')); }); };