[ML] Extract apiDoc params from the schema definitions (#62933)

* [ML] WIP apiDoc schema extractor

* [ML] extract actual type

* [ML] refactor schema definitions

* [ML] Update README.md

* [ML] extract nested

* [ML] call job validation endpoint with complete payload

* [ML] escape special chars and fix line breaks

* [ML] clean up extractDocEntries

* [ML] serializeWithType

* [ML] add missing annotations

* [ML] fix parent schema assigment

* [ML] support object composition

* [ML] support multiple schemas per block

* [ML] fix for collections

* [ML] fix calendarIdsSchema

* [ML] add ml package.json with apidoc commands

* [ML] use the single output markdown file

* [ML] fix typo

* [ML] change the Calendars order

* [ML] adjust the order in adidoc.json

* [ML] update api version

* [ML] update tsconfig.json include

* [ML] update packages/kbn-pm/dist/index.js

* [ML] update ML overrides in .eslintrc.js

* [ML] yarn.lock symlink

* Revert "[ML] yarn.lock symlink"

This reverts commit 07f0680

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.eslintrc.js

@@ -536,9 +536,15 @@ module.exports = {
      * ML overrides
      */
     {
-      files: ['x-pack/legacy/plugins/ml/**/*.js'],
+      files: ['x-pack/plugins/ml/**/*.js'],
       rules: {
         'no-shadow': 'error',
+        'import/no-extraneous-dependencies': [
+          'error',
+          {
+            packageDir: './x-pack',
+          },
+        ],
       },
     },
 

packages/kbn-pm/dist/index.js

@@ -42623,28 +42623,21 @@ module.exports = require("tty");
 const os = __webpack_require__(11);
 const hasFlag = __webpack_require__(12);
 
-const {env} = process;
+const env = process.env;
 
 let forceColor;
 if (hasFlag('no-color') ||
 	hasFlag('no-colors') ||
-	hasFlag('color=false') ||
-	hasFlag('color=never')) {
-	forceColor = 0;
+	hasFlag('color=false')) {
+	forceColor = false;
 } else if (hasFlag('color') ||
 	hasFlag('colors') ||
 	hasFlag('color=true') ||
 	hasFlag('color=always')) {
-	forceColor = 1;
+	forceColor = true;
 }
 if ('FORCE_COLOR' in env) {
-	if (env.FORCE_COLOR === true || env.FORCE_COLOR === 'true') {
-		forceColor = 1;
-	} else if (env.FORCE_COLOR === false || env.FORCE_COLOR === 'false') {
-		forceColor = 0;
-	} else {
-		forceColor = env.FORCE_COLOR.length === 0 ? 1 : Math.min(parseInt(env.FORCE_COLOR, 10), 3);
-	}
+	forceColor = env.FORCE_COLOR.length === 0 || parseInt(env.FORCE_COLOR, 10) !== 0;
 }
 
 function translateLevel(level) {
@@ -42661,7 +42654,7 @@ function translateLevel(level) {
 }
 
 function supportsColor(stream) {
-	if (forceColor === 0) {
+	if (forceColor === false) {
 		return 0;
 	}
 
@@ -42675,15 +42668,11 @@ function supportsColor(stream) {
 		return 2;
 	}
 
-	if (stream && !stream.isTTY && forceColor === undefined) {
+	if (stream && !stream.isTTY && forceColor !== true) {
 		return 0;
 	}
 
-	const min = forceColor || 0;
-
-	if (env.TERM === 'dumb') {
-		return min;
-	}
+	const min = forceColor ? 1 : 0;
 
 	if (process.platform === 'win32') {
 		// Node.js 7.5.0 is the first version of Node.js to include a patch to
@@ -42744,6 +42733,10 @@ function supportsColor(stream) {
 		return 1;
 	}
 
+	if (env.TERM === 'dumb') {
+		return min;
+	}
+
 	return min;
 }
 

x-pack/plugins/ml/package.json

@@ -0,0 +1,15 @@
+{
+  "author": "Elastic",
+  "name": "ml",
+  "version": "0.0.0",
+  "private": true,
+  "license": "Elastic-License",
+  "scripts": {
+    "build:apiDocScripts": "cd server/routes/apidoc_scripts && tsc",
+    "apiDocs": "yarn build:apiDocScripts && cd ./server/routes/ && apidoc --parse-workers apischema=./apidoc_scripts/target/schema_worker.js --parse-parsers apischema=./apidoc_scripts/target/schema_parser.js --parse-filters apiversion=./apidoc_scripts/target/version_filter.js  -i . -o ../routes_doc && apidoc-markdown -p ../routes_doc -o ../routes_doc/ML_API.md"
+  },
+  "devDependencies": {
+    "apidoc": "^0.20.1",
+    "apidoc-markdown": "^5.0.0"
+  }
+}

x-pack/plugins/ml/server/routes/README.md

@@ -6,11 +6,14 @@ Each route handler requires [apiDoc](https://github.com/apidoc/apidoc) annotatio
 to generate documentation.
 The [apidoc-markdown](https://github.com/rigwild/apidoc-markdown) package is also required in order to generate the markdown.
 
-For now the process is pretty manual. You need to make sure the packages mentioned above are installed globally 
-to execute the following command from the directory in which this README file is located.
+There are custom parser and worker (`x-pack/plugins/ml/server/routes/apidoc_scripts`) to process api schemas for each documentation entry. It's written with typescript so make sure all the scripts in the folder are compiled before executing `apidoc` command.
+
+Make sure you have run `yarn kbn bootstrap` to get all requires dev dependencies. Then execute the following command from the ml plugin folder: 
 ```
-apidoc -i . -o ../routes_doc && apidoc-markdown -p ../routes_doc -o ../routes_doc/ML_API.md 
+yarn run apiDocs
 ```
+It compiles all the required scripts and generates the documentation both in HTML and Markdown formats.
+
 
 It will create a new directory `routes_doc` (next to the `routes` folder) which contains the documentation in HTML format 
-as well as `ML_API.md` file.
+as well as `ML_API.md` file.

x-pack/plugins/ml/server/routes/annotations.ts

@@ -5,10 +5,8 @@
  */
 
 import Boom from 'boom';
-import _ from 'lodash';
 import { i18n } from '@kbn/i18n';
 
-import { schema } from '@kbn/config-schema';
 import { SecurityPluginSetup } from '../../../security/server';
 import { isAnnotationsFeatureAvailable } from '../lib/check_annotations';
 import { annotationServiceProvider } from '../models/annotation_service';
@@ -45,10 +43,7 @@ export function annotationRoutes(
    * @apiName GetAnnotations
    * @apiDescription Gets annotations.
    *
-   * @apiParam {String[]} jobIds List of job IDs
-   * @apiParam {String} earliestMs
-   * @apiParam {Number} latestMs
-   * @apiParam {Number} maxAnnotations Max limit of annotations returned
+   * @apiSchema (body) getAnnotationsSchema
    *
    * @apiSuccess {Boolean} success
    * @apiSuccess {Object} annotations
@@ -57,7 +52,7 @@ export function annotationRoutes(
     {
       path: '/api/ml/annotations',
       validate: {
-        body: schema.object(getAnnotationsSchema),
+        body: getAnnotationsSchema,
       },
     },
     mlLicense.fullLicenseAPIGuard(async (context, request, response) => {
@@ -83,14 +78,13 @@ export function annotationRoutes(
    * @apiName IndexAnnotations
    * @apiDescription Index the annotation.
    *
-   * @apiParam {Object} annotation
-   * @apiParam {String} username
+   * @apiSchema (body) indexAnnotationSchema
    */
   router.put(
     {
       path: '/api/ml/annotations/index',
       validate: {
-        body: schema.object(indexAnnotationSchema),
+        body: indexAnnotationSchema,
       },
     },
     mlLicense.fullLicenseAPIGuard(async (context, request, response) => {
@@ -124,17 +118,17 @@ export function annotationRoutes(
   /**
    * @apiGroup Annotations
    *
-   * @api {delete} /api/ml/annotations/index Deletes annotation
+   * @api {delete} /api/ml/annotations/delete/:annotationId Deletes annotation
    * @apiName DeleteAnnotation
    * @apiDescription Deletes specified annotation
    *
-   * @apiParam {String} annotationId
+   * @apiSchema (params) deleteAnnotationSchema
    */
   router.delete(
     {
       path: '/api/ml/annotations/delete/{annotationId}',
       validate: {
-        params: schema.object(deleteAnnotationSchema),
+        params: deleteAnnotationSchema,
       },
     },
     mlLicense.fullLicenseAPIGuard(async (context, request, response) => {