Add output form for csv-on-the-fly

civicrm · May 3, 2022 · 33c19f0 · 33c19f0
1 parent 9efe4c2
commit 33c19f0
Show file tree

Hide file tree

Showing 9 changed files with 151 additions and 51 deletions.
diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php
@@ -60,11 +60,9 @@ public function preProcess() {
       $this->set('tag', $tag);
     }
 
-    if ($invalidRowCount) {
-      $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact';
-      $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
-    }
+    $this->set('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR));
 
+    // @todo conflict rows are still being output in the parser & not updating the temp table - fix
     if ($conflictRowCount) {
       $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact';
       $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
@@ -280,9 +278,7 @@ public function postProcess() {
 
       $this->set('errorFile', $errorFile);
 
-      $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser_Contact';
-      $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
-
+      // @todo - these should use the new url but are not reliably updating the table yet.
       $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser_Contact';
       $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
 

diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php
@@ -27,7 +27,7 @@ public function preProcess() {
     // set the error message path to display
     $this->assign('errorFile', $this->get('errorFile'));
 
-    $totalRowCount = $this->get('totalRowCount');
+    $totalRowCount = $this->getRowCount();
     $relatedCount = $this->get('relatedCount');
     $totalRowCount += $relatedCount;
 
@@ -81,9 +81,6 @@ public function preProcess() {
     $this->assign('dupeActionString', $dupeActionString);
 
     $properties = [
-      'totalRowCount',
-      'validRowCount',
-      'invalidRowCount',
       'conflictRowCount',
       'downloadConflictRecordsUrl',
       'downloadErrorRecordsUrl',
@@ -98,6 +95,10 @@ public function preProcess() {
     foreach ($properties as $property) {
       $this->assign($property, $this->get($property));
     }
+    $this->assign('totalRowCount', $this->getRowCount());
+    $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID));
+    $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
+    $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE));
 
     $session = CRM_Core_Session::singleton();
     $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1'));

diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php
@@ -2739,15 +2739,6 @@ public function run(
         }
       }
 
-      if ($this->_invalidRowCount) {
-        // removed view url for invlaid contacts
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('Reason'),
-        ], $customHeaders);
-        $this->_errorFileName = self::errorFileName(self::ERROR);
-        self::exportCSV($this->_errorFileName, $headers, $this->_errors);
-      }
       if ($this->_conflictCount) {
         $headers = array_merge([
           ts('Line Number'),
@@ -2756,15 +2747,6 @@ public function run(
         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
       }
-      if ($this->_duplicateCount) {
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('View Contact URL'),
-        ], $customHeaders);
-
-        $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
-        self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
-      }
       if ($this->_unMatchCount) {
         $headers = array_merge([
           ts('Line Number'),
@@ -3081,9 +3063,6 @@ public function set($store, $mode = self::MODE_SUMMARY) {
         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
     }
 
-    if ($this->_invalidRowCount) {
-      $store->set('errorsFileName', $this->_errorFileName);
-    }
     if ($this->_conflictCount) {
       $store->set('conflictsFileName', $this->_conflictFileName);
     }
@@ -3101,11 +3080,7 @@ public function set($store, $mode = self::MODE_SUMMARY) {
       if ($this->_duplicateCount) {
         $store->set('duplicatesFileName', $this->_duplicateFileName);
       }
-      if ($this->_unparsedAddressCount) {
-        $store->set('errorsFileName', $this->_errorFileName);
-      }
     }
-    //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
   }
 
   /**

diff --git a/CRM/Core/xml/Menu/Import.xml b/CRM/Core/xml/Menu/Import.xml
@@ -17,6 +17,12 @@
      <page_callback>CRM_Contact_Import_Controller</page_callback>
      <weight>410</weight>
   </item>
+  <item>
+    <path>civicrm/import/outcome</path>
+    <title>Import results</title>
+    <access_arguments>access CiviCRM</access_arguments>
+    <page_callback>CRM_Import_Forms::outputCSV</page_callback>
+  </item>
   <item>
      <path>civicrm/import/activity</path>
      <title>Import Activities</title>

diff --git a/CRM/Import/DataSource.php b/CRM/Import/DataSource.php
@@ -200,10 +200,8 @@ public function getRow(): ?array {
    * @throws \CRM_Core_Exception
    */
   public function getRowCount(array $statuses = []): int {
-    $query = 'SELECT count(*) FROM ' . $this->getTableName();
-    if (!empty($statuses)) {
-      $query .= ' WHERE _status IN (' . implode(',', $statuses) . ')';
-    }
+    $this->statuses = $statuses;
+    $query = 'SELECT count(*) FROM ' . $this->getTableName() . ' ' . $this->getStatusClause();
     return CRM_Core_DAO::singleValueQuery($query);
   }
 
@@ -418,14 +416,40 @@ protected function addTrackingFieldsToTable(string $tableName): void {
    * @throws \CRM_Core_Exception
    */
   private function instantiateQueryObject(): void {
-    $query = 'SELECT * FROM ' . $this->getTableName();
-    if (!empty($this->statuses)) {
-      $query .= ' WHERE _status IN (' . implode(',', $this->statuses) . ')';
-    }
+    $query = 'SELECT * FROM ' . $this->getTableName() . ' ' . $this->getStatusClause();
     if ($this->limit) {
       $query .= ' LIMIT ' . $this->limit . ($this->offset ? (' OFFSET ' . $this->offset) : NULL);
     }
     $this->queryResultObject = CRM_Core_DAO::executeQuery($query);
   }
 
+  /**
+   * Get the mapping of constants to database status codes.
+   *
+   * @return string[]
+   */
+  protected function getStatusMapping() {
+    return [
+      CRM_Import_Parser::VALID => 'imported',
+      CRM_Import_Parser::ERROR => 'error',
+      CRM_Import_Parser::DUPLICATE => 'duplicate',
+    ];
+  }
+
+  /**
+   * Get the status filter clause.
+   *
+   * @return string
+   */
+  private function getStatusClause(): string {
+    if (!empty($this->statuses)) {
+      $statuses = [];
+      foreach ($this->statuses as $status) {
+        $statuses[] = '"' . $this->getStatusMapping()[$status] . '"';
+      }
+      return ' WHERE _status IN (' . implode(',', $statuses) . ')';
+    }
+    return '';
+  }
+
 }
diff --git a/CRM/Import/Form/Summary.php b/CRM/Import/Form/Summary.php
@@ -21,7 +21,7 @@
  * TODO: CRM-11254 - if preProcess and postProcess functions can be reconciled between the 5 child classes,
  * those classes can be removed entirely and this class will not need to be abstract
  */
-abstract class CRM_Import_Form_Summary extends CRM_Core_Form {
+abstract class CRM_Import_Form_Summary extends CRM_Import_Forms {
 
   /**
    * Build the form object.

diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php
@@ -16,6 +16,7 @@
  */
 
 use Civi\Api4\UserJob;
+use League\Csv\Writer;
 
 /**
  * This class helps the forms within the import flow access submitted & parsed values.
@@ -438,14 +439,111 @@ protected function getNumberOfColumns(): int {
    * supporting offset as well.
    *
    * @param int $limit
+   * @param int $offset
+   * @param int|array $statuses
+   *   Optional status filter.
    *
    * @return array
    *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   */
+  protected function getDataRows(int $limit, $offset = 0, $statuses = []): array {
+    $statuses = (array) $statuses;
+    return $this->getDataSourceObject()->getRows($limit, $offset, $statuses);
+  }
+
+  /**
+   * Get x data rows from the datasource.
+   *
+   * At this stage we are fetching from what has been stored in the form
+   * during `postProcess` on the DataSource form.
+   *
+   * In the future we will use the dataSource object, likely
+   * supporting offset as well.
+   *
+   * @return array|int
+   *   One or more of the statues available - e.g
+   *   CRM_Import_Parser::VALID
+   *   or [CRM_Import_Parser::ERROR, CRM_Import_Parser::CONFLICT]
+   *
    * @throws \CRM_Core_Exception
    * @throws \API_Exception
    */
-  protected function getDataRows(int $limit): array {
-    return $this->getDataSourceObject()->getRows($limit);
+  protected function getRowCount($statuses = []): int {
+    $statuses = (array) $statuses;
+    return $this->getDataSourceObject()->getRowCount($statuses);
+  }
+
+  /**
+   * Outputs and downloads the csv of outcomes from an import job.
+   *
+   * This gets the rows from the temp table that match the relevant status
+   * and output them as a csv.
+   *
+   * @throws \API_Exception
+   * @throws \League\Csv\CannotInsertRecord
+   * @throws \CRM_Core_Exception
+   */
+  public static function outputCSV(): void {
+    $userJobID = CRM_Utils_Request::retrieveValue('user_job_id', 'Integer', NULL, TRUE);
+    $status = CRM_Utils_Request::retrieveValue('status', 'String', NULL, TRUE);
+    $saveFileName = CRM_Import_Parser::saveFileName($status);
+
+    $form = new CRM_Import_Forms();
+    $form->controller = new CRM_Core_Controller();
+    $form->set('user_job_id', $userJobID);
+
+    $form->getUserJob();
+    $writer = Writer::createFromFileObject(new SplTempFileObject());
+    $headers = $form->getColumnHeaders();
+    if ($headers) {
+      array_unshift($headers, ts('Reason'));
+      array_unshift($headers, ts('Line Number'));
+      $writer->insertOne($headers);
+    }
+    $writer->addFormatter(['CRM_Import_Forms', 'reorderOutput']);
+    // Note this might be more inefficient that iterating the result
+    // set & doing insertOne - possibly something to explore later.
+    $writer->insertAll($form->getDataRows(0, 0, $status));
+
+    CRM_Utils_System::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0');
+    CRM_Utils_System::setHttpHeader('Content-Description', 'File Transfer');
+    CRM_Utils_System::setHttpHeader('Content-Type', 'text/csv; charset=UTF-8');
+    $writer->output($saveFileName);
+    CRM_Utils_System::civiExit();
+  }
+
+  /**
+   * When outputting the row as a csv, more the last 2 rows to the start.
+   *
+   * This is because the id and status message fields are at the end. It may make sense
+   * to move them to the start later, when order code cleanup has happened...
+   *
+   * @param array $record
+   */
+  public static function reorderOutput(array $record): array {
+    $rowNumber = array_pop($record);
+    $message = array_pop($record);
+    // Also pop off the status - but we are not going to use this at this stage.
+    array_pop($record);
+    array_unshift($record, $message);
+    array_unshift($record, $rowNumber);
+    return $record;
+  }
+
+  /**
+   * Get the url to download the relevant csv file.
+   * @param string $status
+   *
+   * @return string
+   */
+  protected function getDownloadURL(string $status): string {
+    return CRM_Utils_System::url('civicrm/import/outcome', [
+      'user_job_id' => $this->get('user_job_id'),
+      'status' => $status,
+      'reset' => 1,
+    ]);
   }
 
   /**

diff --git a/templates/CRM/Contact/Import/Form/Preview.tpl b/templates/CRM/Contact/Import/Form/Preview.tpl
@@ -20,7 +20,7 @@
 
     {if $invalidRowCount}
         <p class="error">
-        {ts 1=$invalidRowCount 2=$downloadErrorRecordsUrl}CiviCRM has detected invalid data or formatting errors in %1 records. If you continue, these records will be skipped. OR, you can download a file with just these problem records - <a href='%2'>Download Errors</a>. Then correct them in the original import file, cancel this import and begin again at step 1.{/ts}
+        {ts 1=$invalidRowCount 2=$downloadErrorRecordsUrl|smarty:nodefaults}CiviCRM has detected invalid data or formatting errors in %1 records. If you continue, these records will be skipped. OR, you can download a file with just these problem records - <a href='%2'>Download Errors</a>. Then correct them in the original import file, cancel this import and begin again at step 1.{/ts}
         </p>
     {/if}
 
@@ -48,7 +48,7 @@
         <td class="data">{$invalidRowCount}</td>
         <td class="explanation">{ts}Rows with invalid data in one or more fields (for example, invalid email address formatting). These rows will be skipped (not imported).{/ts}
             {if $invalidRowCount}
-                <div class="action-link"><a href="{$downloadErrorRecordsUrl}"><i class="crm-i fa-download" aria-hidden="true"></i> {ts}Download Errors{/ts}</a></div>
+                <div class="action-link"><a href="{$downloadErrorRecordsUrl|smarty:nodefaults}"><i class="crm-i fa-download" aria-hidden="true"></i> {ts}Download Errors{/ts}</a></div>
             {/if}
         </td>
     </tr>

diff --git a/templates/CRM/Contact/Import/Form/Summary.tpl b/templates/CRM/Contact/Import/Form/Summary.tpl
@@ -33,7 +33,7 @@
         {ts count=$invalidRowCount plural='CiviCRM has detected invalid data and/or formatting errors in %count records. These records have not been imported.'}CiviCRM has detected invalid data and/or formatting errors in one record. This record has not been imported.{/ts}
         </p>
         <p class="error">
-        {ts 1=$downloadErrorRecordsUrl}You can <a href='%1'>Download Errors</a>. You may then correct them, and import the new file with the corrected data.{/ts}
+        {ts 1=$downloadErrorRecordsUrl|smarty:nodefaults}You can <a href='%1'>Download Errors</a>. You may then correct them, and import the new file with the corrected data.{/ts}
         </p>
     {/if}
 
@@ -75,7 +75,7 @@
         <td class="data">{$invalidRowCount}</td>
         <td class="explanation">{ts}Rows with invalid data in one or more fields (for example, invalid email address formatting). These rows will be skipped (not imported).{/ts}
             {if $invalidRowCount}
-                <div class="action-link"><a href="{$downloadErrorRecordsUrl}"><i class="crm-i fa-download" aria-hidden="true"></i> {ts}Download Errors{/ts}</a></div>
+                <div class="action-link"><a href="{$downloadErrorRecordsUrl|smarty:nodefaults}"><i class="crm-i fa-download" aria-hidden="true"></i> {ts}Download Errors{/ts}</a></div>
             {/if}
         </td>
     </tr>