feat: add data update bulk/resume commands (#1098)

* chore: refactor bulk ingest utils

* feat: add `data update bulk/resume`

* fix: update `data import bulk` help

* test: add bulk update NUT

* test: break up NUTs (#1099)

* chore: unify bulk ingest logic

* test: add bulk update NUTs to test matrix

* fix: insert operation

* fix: command-specific resume instructions

* fix: command-specific stage title

* fix: pass operation opt

* test: fix update resume NUT on win

* test: refactor/doc

* chore: moar refactor/doc

* chore: clean up msgs

* feat: add column-delimiter flag to import/update bulk

* chore: update command snapshot

* chore: eslint rule inline

* test: validate async command's cache files

* chore: update msg

[skip ci]

* fix: edit help for new "data update bulk|resume" commands (#1106)

* fix: remove `as string`

* chore: use proper stop status

* chore: share column-delimiter flag def

* test: remove type assertions

* feat: detect column delimiter

* test: nut should detect column delimiter

---------

Co-authored-by: Juliet Shackell <63259011+jshackell-sfdc@users.noreply.github.com>

.github/workflows/test.yml

@@ -21,6 +21,17 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
+        command:
+          - 'yarn test:nuts:bulk:export'
+          - 'yarn test:nuts:bulk:import'
+          - 'yarn test:nuts:bulk:update'
+          - 'yarn test:nuts:data:bulk-upsert-delete'
+          - 'yarn test:nuts:data:create'
+          - 'yarn test:nuts:data:query'
+          - 'yarn test:nuts:data:record'
+          - 'yarn test:nuts:data:search'
+          - 'yarn test:nuts:data:tree'
       fail-fast: false
     with:
       os: ${{ matrix.os }}
+      command: ${{ matrix.command }}

command-snapshot.json

@@ -142,7 +142,18 @@
     "command": "data:import:bulk",
     "flagAliases": [],
     "flagChars": ["a", "f", "o", "s", "w"],
-    "flags": ["api-version", "async", "file", "flags-dir", "json", "line-ending", "sobject", "target-org", "wait"],
+    "flags": [
+      "api-version",
+      "async",
+      "column-delimiter",
+      "file",
+      "flags-dir",
+      "json",
+      "line-ending",
+      "sobject",
+      "target-org",
+      "wait"
+    ],
     "plugin": "@salesforce/plugin-data"
   },
   {
@@ -235,6 +246,25 @@
     "flags": ["api-version", "file", "flags-dir", "json", "query", "result-format", "target-org"],
     "plugin": "@salesforce/plugin-data"
   },
+  {
+    "alias": [],
+    "command": "data:update:bulk",
+    "flagAliases": [],
+    "flagChars": ["a", "f", "o", "s", "w"],
+    "flags": [
+      "api-version",
+      "async",
+      "column-delimiter",
+      "file",
+      "flags-dir",
+      "json",
+      "line-ending",
+      "sobject",
+      "target-org",
+      "wait"
+    ],
+    "plugin": "@salesforce/plugin-data"
+  },
   {
     "alias": ["force:data:record:update"],
     "command": "data:update:record",
@@ -255,6 +285,14 @@
     ],
     "plugin": "@salesforce/plugin-data"
   },
+  {
+    "alias": [],
+    "command": "data:update:resume",
+    "flagAliases": [],
+    "flagChars": ["i", "w"],
+    "flags": ["flags-dir", "job-id", "json", "use-most-recent", "wait"],
+    "plugin": "@salesforce/plugin-data"
+  },
   {
     "alias": [],
     "command": "data:upsert:bulk",

messages/bulkIngest.md

@@ -0,0 +1,39 @@
+# export.resume
+
+Run "sf %s --job-id %s" to resume the operation.
+
+# error.timeout
+
+The operation timed out after %s minutes.
+
+Run "sf %s --job-id %s" to resume it.
+
+# error.failedRecordDetails
+
+Job finished being processed but failed to process %s records.
+
+To review the details of this job, run this command:
+
+sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
+
+# error.jobFailed
+
+Job failed to be processed due to:
+
+%s
+
+To review the details of this job, run this command:
+
+sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
+
+# error.jobAborted
+
+Job has been aborted.
+
+To review the details of this job, run this command:
+
+sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
+
+# flags.column-delimiter.summary
+
+Column delimiter used in the CSV file. Default is COMMA.

messages/data.export.resume.md

@@ -1,6 +1,6 @@
 # summary
 
-Resume a bulk export job that you previously started.
+Resume a bulk export job that you previously started. Uses Bulk API 2.0.
 
 # description
 

messages/data.import.bulk.md

@@ -40,40 +40,8 @@ Time to wait for the command to finish, in minutes.
 
 # flags.line-ending.summary
 
-Line ending used in the CSV file. Default value on Windows is `CRLF`; on macOS and Linux it's `LR`.
+Line ending used in the CSV file. Default value on Windows is `CRLF`; on macOS and Linux it's `LF`.
 
-# export.resume
+# flags.column-delimiter.summary
 
-Run "sf data import resume --job-id %s" to resume the operation.
-
-# error.timeout
-
-The operation timed out after %s minutes.
-
-Run "sf data import resume --job-id %s" to resume it.
-
-# error.failedRecordDetails
-
-Job finished being processed but failed to import %s records.
-
-To review the details of this job, run this command:
-
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
-
-# error.jobFailed
-
-Job failed to be processed due to:
-
-%s
-
-To review the details of this job, run this command:
-
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
-
-# error.jobAborted
-
-Job has been aborted.
-
-To review the details of this job, run this command:
-
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
+Column delimiter used in the CSV file. Default is COMMA.

messages/data.import.resume.md

@@ -27,35 +27,3 @@ Job ID of the bulk import.
 # flags.wait.summary
 
 Time to wait for the command to finish, in minutes.
-
-# error.failedRecordDetails
-
-Job finished being processed but failed to import %s records.
-
-To review the details of this job, run this command:
-
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
-
-# error.timeout
-
-The operation timed out after %s minutes.
-
-Try re-running "sf data import resume --job-id %s" with a bigger wait time.
-
-# error.jobFailed
-
-Job failed to be processed due to:
-
-%s
-
-To review the details of this job, run this command:
-
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
-
-# error.jobAborted
-
-Job has been aborted.
-
-To review the details of this job, run this command:
-
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"

messages/data.update.bulk.md

@@ -0,0 +1,47 @@
+# summary
+
+Bulk update records to an org from a CSV file. Uses Bulk API 2.0.
+
+# description
+
+You can use this command to update millions of Salesforce object records based on a file in comma-separated values (CSV) format.
+
+All the records in the CSV file must be for the same Salesforce object. Specify the object with the `--sobject` flag. The first column of every line in the CSV file must be an ID of the record you want to update. The CSV file can contain only existing records; if a record in the file doesn't currently exist in the Salesforce object, the command fails. Consider using "sf data upsert bulk" if you also want to insert new records.
+
+Bulk updates can take a while, depending on how many records are in the CSV file. If the command times out, or you specified the --async flag, the command displays the job ID. To see the status and get the results of the job, run "sf data update resume" and pass the job ID to the --job-id flag.
+
+For information and examples about how to prepare your CSV files, see "Prepare Data to Ingest" in the "Bulk API 2.0 and Bulk API Developer Guide" (https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/datafiles_prepare_data.htm).
+
+# examples
+
+- Update Account records from a CSV-formatted file into an org with alias "my-scratch"; if the update doesn't complete in 10 minutes, the command ends and displays a job ID:
+
+  <%= config.bin %> <%= command.id %> --file accounts.csv --sobject Account --wait 10 --target-org my-scratch
+
+- Update asynchronously and use the default org; the command immediately returns a job ID that you then pass to the "sf data update resume" command:
+
+  <%= config.bin %> <%= command.id %> --file accounts.csv --sobject Account --async
+
+# flags.async.summary
+
+Don't wait for the command to complete.
+
+# flags.wait.summary
+
+Time to wait for the command to finish, in minutes.
+
+# flags.file.summary
+
+CSV file that contains the Salesforce object records you want to update.
+
+# flags.sobject.summary
+
+API name of the Salesforce object, either standard or custom, which you are updating.
+
+# flags.line-ending.summary
+
+Line ending used in the CSV file. Default value on Windows is `CRLF`; on macOS and Linux it's `LF`.
+
+# flags.column-delimiter.summary
+
+Column delimiter used in the CSV file. Default is COMMA.

messages/data.update.resume.md

@@ -0,0 +1,29 @@
+# summary
+
+Resume a bulk update job that you previously started. Uses Bulk API 2.0.
+
+# description
+
+When the original "sf data update bulk" command either times out or is run with the --async flag, it displays a job ID. To see the status and get the results of the bulk update, run this command by either passing it the job ID or using the --use-most-recent flag to specify the most recent bulk update job.
+
+# examples
+
+- Resume a bulk update job of your default org using a job ID:
+
+  <%= config.bin %> <%= command.id %> --job-id 750xx000000005sAAA
+
+- Resume the most recently run bulk update job for an org with alias "my-scratch":
+
+  <%= config.bin %> <%= command.id %> --use-most-recent --target-org my-scratch
+
+# flags.use-most-recent.summary
+
+Use the job ID of the bulk update job that was most recently run.
+
+# flags.job-id.summary
+
+Job ID of the bulk update.
+
+# flags.wait.summary
+
+Time to wait for the command to finish, in minutes.

package.json

@@ -67,7 +67,8 @@
             "description": "Query records."
           },
           "update": {
-            "description": "Update a single record."
+            "description": "Update many records.",
+            "external": true
           },
           "upsert": {
             "description": "Upsert many records."
@@ -103,8 +104,15 @@
     "prepack": "sf-prepack",
     "prepare": "sf-install",
     "test": "wireit",
-    "test:nuts": "nyc mocha \"./test/**/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
-    "test:nuts:bulk": "nyc mocha \"./test/**/dataBulk.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:bulk:import": "nyc mocha \"./test/commands/data/import/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:bulk:export": "nyc mocha \"./test/commands/data/export/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:bulk:update": "nyc mocha \"./test/commands/data/update/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:data:tree": "nyc mocha \"./test/commands/data/tree/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:data:query": "nyc mocha \"./test/commands/data/query/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:data:record": "nyc mocha \"./test/commands/data/record/dataRecord.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:data:search": "nyc mocha \"./test/commands/data/search.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:data:create": "nyc mocha \"./test/commands/data/create/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:data:bulk-upsert-delete": "nyc mocha \"./test/commands/data/dataBulk.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
     "test:only": "wireit",
     "version": "oclif readme"
   },

src/bulkDataRequestCache.ts

@@ -257,6 +257,75 @@ export class BulkImportRequestCache extends TTLConfig<TTLConfig.Options, BulkExp
   }
 }
 
+export class BulkUpdateRequestCache extends TTLConfig<TTLConfig.Options, BulkExportCacheConfig> {
+  public static getDefaultOptions(): TTLConfig.Options {
+    return {
+      isGlobal: true,
+      isState: true,
+      filename: BulkUpdateRequestCache.getFileName(),
+      stateFolder: Global.SF_STATE_FOLDER,
+      ttl: Duration.days(7),
+    };
+  }
+
+  public static getFileName(): string {
+    return 'bulk-data-update-cache.json';
+  }
+
+  public static async unset(key: string): Promise<void> {
+    const cache = await BulkImportRequestCache.create();
+    cache.unset(key);
+    await cache.write();
+  }
+
+  /**
+   * Creates a new bulk data import cache entry for the given bulk request id.
+   *
+   * @param bulkRequestId
+   * @param username
+   * @param apiVersion
+   */
+  public async createCacheEntryForRequest(bulkRequestId: string, username: string, apiVersion: string): Promise<void> {
+    this.set(bulkRequestId, {
+      jobId: bulkRequestId,
+      username,
+      apiVersion,
+    });
+    await this.write();
+    Logger.childFromRoot('BulkUpdateCache').debug(`bulk cache saved for ${bulkRequestId}`);
+  }
+
+  public async resolveResumeOptionsFromCache(jobIdOrMostRecent: string | boolean): Promise<ResumeBulkImportOptions> {
+    if (typeof jobIdOrMostRecent === 'boolean') {
+      const key = this.getLatestKey();
+      if (!key) {
+        throw messages.createError('error.missingCacheEntryError');
+      }
+      // key definitely exists because it came from the cache
+      const entry = this.get(key);
+
+      return {
+        jobInfo: { id: entry.jobId },
+        options: {
+          connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(),
+        },
+      };
+    } else {
+      const entry = this.get(jobIdOrMostRecent);
+      if (!entry) {
+        throw messages.createError('error.bulkRequestIdNotFound', [jobIdOrMostRecent]);
+      }
+
+      return {
+        jobInfo: { id: entry.jobId },
+        options: {
+          connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(),
+        },
+      };
+    }
+  }
+}
+
 export class BulkExportRequestCache extends TTLConfig<TTLConfig.Options, BulkExportCacheConfig> {
   public static getDefaultOptions(): TTLConfig.Options {
     return {