Improve es-archiver saving/loading (elastic#118255)

fkanout · Nov 17, 2021 · 7c81857 · 7c81857
1 parent 04fb72b
commit 7c81857
Show file tree

Hide file tree

Showing 12 changed files with 377 additions and 93 deletions.
diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts
@@ -40,13 +40,15 @@ export async function loadAction({
   inputDir,
   skipExisting,
   useCreate,
+  docsOnly,
   client,
   log,
   kbnClient,
 }: {
   inputDir: string;
   skipExisting: boolean;
   useCreate: boolean;
+  docsOnly?: boolean;
   client: Client;
   log: ToolingLog;
   kbnClient: KbnClient;
@@ -76,7 +78,7 @@ export async function loadAction({
 
   await createPromiseFromStreams([
     recordStream,
-    createCreateIndexStream({ client, stats, skipExisting, log }),
+    createCreateIndexStream({ client, stats, skipExisting, docsOnly, log }),
     createIndexDocRecordsStream(client, stats, progress, useCreate),
   ]);
 

diff --git a/packages/kbn-es-archiver/src/actions/save.ts b/packages/kbn-es-archiver/src/actions/save.ts
@@ -27,13 +27,15 @@ export async function saveAction({
   client,
   log,
   raw,
+  keepIndexNames,
   query,
 }: {
   outputDir: string;
   indices: string | string[];
   client: Client;
   log: ToolingLog;
   raw: boolean;
+  keepIndexNames?: boolean;
   query?: Record<string, any>;
 }) {
   const name = relative(REPO_ROOT, outputDir);
@@ -50,15 +52,15 @@ export async function saveAction({
     // export and save the matching indices to mappings.json
     createPromiseFromStreams([
       createListStream(indices),
-      createGenerateIndexRecordsStream(client, stats),
+      createGenerateIndexRecordsStream({ client, stats, keepIndexNames }),
       ...createFormatArchiveStreams(),
       createWriteStream(resolve(outputDir, 'mappings.json')),
     ] as [Readable, ...Writable[]]),
 
     // export all documents from matching indexes into data.json.gz
     createPromiseFromStreams([
       createListStream(indices),
-      createGenerateDocRecordsStream({ client, stats, progress, query }),
+      createGenerateDocRecordsStream({ client, stats, progress, keepIndexNames, query }),
       ...createFormatArchiveStreams({ gzip: !raw }),
       createWriteStream(resolve(outputDir, `data.json${raw ? '' : '.gz'}`)),
     ] as [Readable, ...Writable[]]),

diff --git a/packages/kbn-es-archiver/src/cli.ts b/packages/kbn-es-archiver/src/cli.ts
@@ -143,11 +143,12 @@ export function runCli() {
           $ node scripts/es_archiver save test/functional/es_archives/my_test_data logstash-*
       `,
       flags: {
-        boolean: ['raw'],
+        boolean: ['raw', 'keep-index-names'],
         string: ['query'],
         help: `
-          --raw              don't gzip the archives
-          --query            query object to limit the documents being archived, needs to be properly escaped JSON
+          --raw                    don't gzip the archives
+          --keep-index-names       don't change the names of Kibana indices to .kibana_1
+          --query                  query object to limit the documents being archived, needs to be properly escaped JSON
         `,
       },
       async run({ flags, esArchiver, statsMeta }) {
@@ -168,6 +169,11 @@ export function runCli() {
           throw createFlagError('--raw does not take a value');
         }
 
+        const keepIndexNames = flags['keep-index-names'];
+        if (typeof keepIndexNames !== 'boolean') {
+          throw createFlagError('--keep-index-names does not take a value');
+        }
+
         const query = flags.query;
         let parsedQuery;
         if (typeof query === 'string' && query.length > 0) {
@@ -178,7 +184,7 @@ export function runCli() {
           }
         }
 
-        await esArchiver.save(path, indices, { raw, query: parsedQuery });
+        await esArchiver.save(path, indices, { raw, keepIndexNames, query: parsedQuery });
       },
     })
     .command({
@@ -196,9 +202,10 @@ export function runCli() {
           $ node scripts/es_archiver load my_test_data --config ../config.js
       `,
       flags: {
-        boolean: ['use-create'],
+        boolean: ['use-create', 'docs-only'],
         help: `
           --use-create       use create instead of index for loading documents
+          --docs-only        load only documents, not indices
         `,
       },
       async run({ flags, esArchiver, statsMeta }) {
@@ -217,7 +224,12 @@ export function runCli() {
           throw createFlagError('--use-create does not take a value');
         }
 
-        await esArchiver.load(path, { useCreate });
+        const docsOnly = flags['docs-only'];
+        if (typeof docsOnly !== 'boolean') {
+          throw createFlagError('--docs-only does not take a value');
+        }
+
+        await esArchiver.load(path, { useCreate, docsOnly });
       },
     })
     .command({

diff --git a/packages/kbn-es-archiver/src/es_archiver.ts b/packages/kbn-es-archiver/src/es_archiver.ts
@@ -50,16 +50,22 @@ export class EsArchiver {
    * @param {String|Array<String>} indices - the indices to archive
    * @param {Object} options
    * @property {Boolean} options.raw - should the archive be raw (unzipped) or not
+   * @property {Boolean} options.keepIndexNames - should the Kibana index name be kept as-is or renamed
    */
   async save(
     path: string,
     indices: string | string[],
-    { raw = false, query }: { raw?: boolean; query?: Record<string, any> } = {}
+    {
+      raw = false,
+      keepIndexNames = false,
+      query,
+    }: { raw?: boolean; keepIndexNames?: boolean; query?: Record<string, any> } = {}
   ) {
     return await saveAction({
       outputDir: Path.resolve(this.baseDir, path),
       indices,
       raw,
+      keepIndexNames,
       client: this.client,
       log: this.log,
       query,
@@ -74,18 +80,21 @@ export class EsArchiver {
    * @property {Boolean} options.skipExisting - should existing indices
    *                                           be ignored or overwritten
    * @property {Boolean} options.useCreate - use a create operation instead of index for documents
+   * @property {Boolean} options.docsOnly - load only documents, not indices
    */
   async load(
     path: string,
     {
       skipExisting = false,
       useCreate = false,
-    }: { skipExisting?: boolean; useCreate?: boolean } = {}
+      docsOnly = false,
+    }: { skipExisting?: boolean; useCreate?: boolean; docsOnly?: boolean } = {}
   ) {
     return await loadAction({
       inputDir: this.findArchive(path),
       skipExisting: !!skipExisting,
       useCreate: !!useCreate,
+      docsOnly,
       client: this.client,
       log: this.log,
       kbnClient: this.kbnClient,