Import - start to use civicrm_user_job to track user imports

CRM/Contact/Import/Form/DataSource.php

@@ -74,7 +74,7 @@ public function preProcess() {
   public function buildQuickForm() {
 
     $this->assign('urlPath', 'civicrm/import/datasource');
-    $this->assign('urlPathVar', 'snippet=4');
+    $this->assign('urlPathVar', 'snippet=4&user_job_id=' . $this->get('user_job_id'));
 
     $this->add('select', 'dataSource', ts('Data Source'), $this->getDataSources(), TRUE,
       ['onchange' => 'buildDataSourceFormBlock(this.value);']
@@ -168,10 +168,16 @@ public function setDefaultValues() {
    * Call the DataSource's postProcess method.
    *
    * @throws \CRM_Core_Exception
+   * @throws \API_Exception
    */
   public function postProcess() {
     $this->controller->resetPage('MapField');
-
+    if (!$this->getUserJobID()) {
+      $this->createUserJob();
+    }
+    else {
+      $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues());
+    }
     // Setup the params array
     $this->_params = $this->controller->exportValues($this->_name);
 
@@ -208,6 +214,7 @@ public function postProcess() {
 
     $parser = new CRM_Contact_Import_Parser_Contact($mapper);
     $parser->setMaxLinesToProcess(100);
+    $parser->setUserJobID($this->getUserJobID());
     $parser->run($importTableName,
       $mapper,
       CRM_Import_Parser::MODE_MAPFIELD,
@@ -234,12 +241,12 @@ public function postProcess() {
    * @throws \CRM_Core_Exception
    */
   private function instantiateDataSource(): void {
-    $dataSourceName = $this->getDataSourceClassName();
-    $dataSource = new $dataSourceName();
+    $dataSource = $this->getDataSourceObject();
     // Get the PEAR::DB object
     $dao = new CRM_Core_DAO();
     $db = $dao->getDatabaseConnection();
     $dataSource->postProcess($this->_params, $db, $this);
+    $this->updateUserJobMetadata('DataSource', $dataSource->getDataSourceMetadata());
   }
 
   /**

CRM/Core/BAO/UserJob.php

@@ -20,6 +20,22 @@
  */
 class CRM_Core_BAO_UserJob extends CRM_Core_DAO_UserJob {
 
+  /**
+   * Restrict access to the relevant user.
+   *
+   * Note that it is likely we might want to permit other users such as
+   * sysadmins to access other people's user_jobs in future but it has been
+   * kept tightly restricted for initial simplicity (ie do we want to
+   * use an existing permission? a new permission ? do they require
+   * 'view all contacts' etc.
+   *
+   * @inheritDoc
+   */
+  public function addSelectWhereClause(): array {
+    $clauses['created_id'] = '= ' . (int) CRM_Core_Session::getLoggedInContactID();
+    return $clauses;
+  }
+
   /**
    * Get the statuses for Import Jobs.
    *
@@ -34,11 +50,16 @@ public static function getStatuses(): array {
       ],
       [
         'id' => 2,
+        'name' => 'draft',
+        'label' => ts('Draft'),
+      ],
+      [
+        'id' => 3,
         'name' => 'scheduled',
         'label' => ts('Scheduled'),
       ],
       [
-        'id' => 3,
+        'id' => 4,
         'name' => 'in_progress',
         'label' => ts('In Progress'),
       ],

CRM/Core/Permission.php

@@ -1543,6 +1543,18 @@ public static function getEntityActionPermissions() {
     $permissions['option_value'] = $permissions['uf_group'];
     $permissions['option_group'] = $permissions['option_value'];
 
+    // User Job permissions - we access these using acls on the get action.
+    // For create it probably makes sense (at least initially) to be stricter
+    // as the forms doing the work can set the permission check to FALSE.
+    $permissions['user_job'] = [
+      'get' => [
+        'access CiviCRM',
+      ],
+      'default' => [
+        'administer CiviCRM',
+      ],
+    ];
+
     $permissions['custom_value'] = [
       'gettree' => ['access CiviCRM'],
     ];

CRM/Import/DataSource.php

@@ -15,12 +15,118 @@
  * @copyright CiviCRM LLC https://civicrm.org/licensing
  */
 
+use Civi\Api4\UserJob;
+
 /**
  * This class defines the DataSource interface but must be subclassed to be
  * useful.
  */
 abstract class CRM_Import_DataSource {
 
+  /**
+   * Class constructor.
+   *
+   * @param int|null $userJobID
+   */
+  public function __construct(int $userJobID = NULL) {
+    if ($userJobID) {
+      $this->setUserJobID($userJobID);
+    }
+  }
+
+  /**
+   * Form fields declared for this datasource.
+   *
+   * @var string[]
+   */
+  protected $submittableFields = [];
+
+  /**
+   * User job id.
+   *
+   * This is the primary key of the civicrm_user_job table which is used to
+   * track the import.
+   *
+   * @var int
+   */
+  protected $userJobID;
+
+  /**
+   * @return int|null
+   */
+  public function getUserJobID(): ?int {
+    return $this->userJobID;
+  }
+
+  /**
+   * Set user job ID.
+   *
+   * @param int $userJobID
+   */
+  public function setUserJobID(int $userJobID): void {
+    $this->userJobID = $userJobID;
+  }
+
+  /**
+   * User job details.
+   *
+   * This is the relevant row from civicrm_user_job.
+   *
+   * @var array
+   */
+  protected $userJob;
+
+  /**
+   * Get User Job.
+   *
+   * API call to retrieve the userJob row.
+   *
+   * @return array
+   *
+   * @throws \API_Exception
+   */
+  protected function getUserJob(): array {
+    if (!$this->userJob) {
+      $this->userJob = UserJob::get()
+        ->addWhere('id', '=', $this->getUserJobID())
+        ->execute()
+        ->first();
+    }
+    return $this->userJob;
+  }
+
+  /**
+   * Generated metadata relating to the the datasource.
+   *
+   * This is values that are computed within the DataSource class and
+   * which are stored in the userJob metadata in the DataSource key - eg.
+   *
+   * ['table_name' => $]
+   *
+   * Will be in the user_job.metadata field encoded into the json like
+   *
+   * `{'DataSource' : ['table_name' => $], 'submitted_values' : .....}`
+   *
+   * @var array
+   */
+  protected $dataSourceMetadata = [];
+
+  /**
+   * @return array
+   */
+  public function getDataSourceMetadata(): array {
+    return $this->dataSourceMetadata;
+  }
+
+  /**
+   * Get the fields declared for this datasource.
+   *
+   * @return string[]
+   */
+  public function getSubmittableFields(): array {
+    return $this->submittableFields;
+  }
+
   /**
    * Provides information about the data source.
    *

CRM/Import/DataSource/CSV.php

@@ -18,6 +18,13 @@ class CRM_Import_DataSource_CSV extends CRM_Import_DataSource {
   const
     NUM_ROWS_TO_INSERT = 100;
 
+  /**
+   * Form fields declared for this datasource.
+   *
+   * @var string[]
+   */
+  protected $submittableFields = ['skipColumnHeader', 'uploadField'];
+
   /**
    * Provides information about the data source.
    *
@@ -88,8 +95,12 @@ public function postProcess(&$params, &$db, &$form) {
       CRM_Utils_Array::value('fieldSeparator', $params, ',')
     );
 
-    $form->set('originalColHeader', CRM_Utils_Array::value('original_col_header', $result));
+    $form->set('originalColHeader', CRM_Utils_Array::value('column_headers', $result));
     $form->set('importTableName', $result['import_table_name']);
+    $this->dataSourceMetadata = [
+      'table_name' => $result['import_table_name'],
+      'column_headers' => $result['column_headers'] ?? NULL,
+    ];
   }
 
   /**
@@ -135,7 +146,7 @@ private static function _CsvToTable(
     // create the column names from the CSV header or as col_0, col_1, etc.
     if ($headers) {
       //need to get original headers.
-      $result['original_col_header'] = $firstrow;
+      $result['column_headers'] = $firstrow;
 
       $strtolower = function_exists('mb_strtolower') ? 'mb_strtolower' : 'strtolower';
       $columns = array_map($strtolower, $firstrow);
@@ -242,7 +253,6 @@ private static function _CsvToTable(
 
     //get the import tmp table name.
     $result['import_table_name'] = $tableName;
-
     return $result;
   }
 

CRM/Import/DataSource/SQL.php

@@ -16,6 +16,13 @@
  */
 class CRM_Import_DataSource_SQL extends CRM_Import_DataSource {
 
+  /**
+   * Form fields declared for this datasource.
+   *
+   * @var string[]
+   */
+  protected $submittableFields = ['sqlQuery'];
+
   /**
    * Provides information about the data source.
    *
@@ -90,6 +97,9 @@ public function postProcess(&$params, &$db, &$form) {
     );
 
     $form->set('importTableName', $importJob->getTableName());
+    $this->dataSourceMetadata = [
+      'table_name' => $importJob->getTableName(),
+    ];
   }
 
 }

CRM/Import/Form/DataSourceConfig.php

@@ -29,13 +29,52 @@ public function preProcess(): void {
     $dataSourcePath = explode('_', $this->getDataSourceClassName());
     $templateFile = 'CRM/Contact/Import/Form/' . $dataSourcePath[3] . '.tpl';
     $this->assign('dataSourceFormTemplateFile', $templateFile ?? NULL);
+    if (CRM_Utils_Request::retrieveValue('user_job_id', 'Integer')) {
+      $this->setUserJobID(CRM_Utils_Request::retrieveValue('user_job_id', 'Integer'));
+    }
   }
 
   /**
    * Build the form object.
+   *
+   * @throws \CRM_Core_Exception
    */
   public function buildQuickForm(): void {
     $this->buildDataSourceFields();
   }
 
+  /**
+   * Set defaults.
+   *
+   * @return array
+   *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   */
+  public function setDefaultValues() {
+    $defaults = [];
+    if ($this->userJobID) {
+      foreach ($this->getDataSourceFields() as $fieldName) {
+        $defaults[$fieldName] = $this->getSubmittedValue($fieldName);
+      }
+    }
+    return $defaults;
+  }
+
+  /**
+   * Get the submitted value, as saved in the user job.
+   *
+   * This form is not in the same flow as the DataSource but
+   * the value we want is saved to the userJob so load it from there.
+   *
+   * @param string $fieldName
+   *
+   * @return mixed|null
+   * @throws \API_Exception
+   */
+  public function getSubmittedValue(string $fieldName) {
+    $userJob = $this->getUserJob();
+    return $userJob['metadata']['submitted_values'][$fieldName];
+  }
+
 }