CRM-21260: CiviMail compose UI very slow to initialize, CRM-21316: Refactor getRecipients fn

CRM/Mailing/BAO/MailingAB.php

@@ -147,7 +147,7 @@ public static function del($id) {
    * @param CRM_Mailing_DAO_MailingAB $dao
    */
   public static function distributeRecipients(CRM_Mailing_DAO_MailingAB $dao) {
-    CRM_Mailing_BAO_Mailing::getRecipients($dao->mailing_id_a, $dao->mailing_id_a, TRUE);
+    CRM_Mailing_BAO_Mailing::getRecipients($dao->mailing_id_a);
 
     //calculate total number of random recipients for mail C from group_percentage selected
     $totalCount = CRM_Mailing_BAO_Recipients::mailingSize($dao->mailing_id_a);

CRM/Mailing/BAO/MailingJob.php

@@ -69,10 +69,8 @@ static public function create($params) {
     $job->scheduled_date = $params['scheduled_date'];
     $job->is_test = $params['is_test'];
     $job->save();
-    $mailing = new CRM_Mailing_BAO_Mailing();
-    $mailing->id = $params['mailing_id'];
-    if ($mailing->id && $mailing->find(TRUE)) {
-      $mailing->getRecipients($job->id, $params['mailing_id'], TRUE, $mailing->dedupe_email);
+    if ($params['mailing_id']) {
+      CRM_Mailing_BAO_Mailing::getRecipients($params['mailing_id']);
       return $job;
     }
     else {

CRM/SMS/Form/Group.php

@@ -266,12 +266,7 @@ public function postProcess() {
     $this->set('mailing_id', $mailing->id);
 
     // also compute the recipients and store them in the mailing recipients table
-    CRM_Mailing_BAO_Mailing::getRecipients($mailing->id,
-      $mailing->id,
-      TRUE,
-      FALSE,
-      'sms'
-    );
+    CRM_Mailing_BAO_Mailing::getRecipients($mailing->id);
 
     $count = CRM_Mailing_BAO_Recipients::mailingSize($mailing->id);
     $this->set('count', $count);

CRM/Utils/SQL/Select.php

@@ -114,6 +114,7 @@ class CRM_Utils_SQL_Select implements ArrayAccess {
 
   private $mode = NULL;
   private $insertInto = NULL;
+  private $insertVerb = 'INSERT INTO ';
   private $insertIntoFields = array();
   private $selects = array();
   private $from;
@@ -403,6 +404,34 @@ public function insertInto($table, $fields = array()) {
     return $this;
   }
 
+  /**
+   * Wrapper function of insertInto fn but sets insertVerb = "INSERT IGNORE INTO "
+   *
+   * @param string $table
+   *   The name of the other table (which receives new data).
+   * @param array $fields
+   *   The fields to fill in the other table (in order).
+   * @return CRM_Utils_SQL_Select
+   */
+  public function insertIgnoreInto($table, $fields = array()) {
+    $this->insertVerb = "INSERT IGNORE INTO ";
+    return $this->insertInto($table, $fields);
+  }
+
+  /**
+   * Wrapper function of insertInto fn but sets insertVerb = "REPLACE INTO "
+   *
+   * @param string $table
+   *   The name of the other table (which receives new data).
+   * @param array $fields
+   *   The fields to fill in the other table (in order).
+   */
+  public function replaceInto($table, $fields = array()) {
+    $this->insertVerb = "REPLACE INTO ";
+    return $this->insertInto($table, $fields);
+  }
+
+
   /**
    * @param array $fields
    *   The fields to fill in the other table (in order).
@@ -550,10 +579,11 @@ public function escapeString($value) {
   public function toSQL() {
     $sql = '';
     if ($this->insertInto) {
-      $sql .= 'INSERT INTO ' . $this->insertInto . ' (';
+      $sql .= $this->insertVerb . $this->insertInto . ' (';
       $sql .= implode(', ', $this->insertIntoFields);
       $sql .= ")\n";
     }
+
     if ($this->selects) {
       $sql .= 'SELECT ' . $this->distinct . implode(', ', $this->selects) . "\n";
     }

ang/crmMailing/Recipients.js

@@ -145,6 +145,9 @@
                 mids.push(dv.entity_id);
               }
             }
+            if (gids.length === 0) {
+              return;
+            }
 
             CRM.api3('Group', 'getlist', { params: { id: { IN: gids }, options: { limit: 0 } }, extra: ["is_hidden"] } ).then(
               function(glist) {

ang/crmMailing/services.js

@@ -270,12 +270,13 @@
         else {
           // Protect against races in saving and previewing by chaining create+preview.
           var params = angular.extend({}, mailing, mailing.recipients, {
-            options: {force_rollback: 1},
+            id: mailing.id,
             'api.Mailing.preview': {
               id: '$value.id'
             }
           });
           delete params.recipients; // the content was merged in
+          params._skip_evil_bao_auto_recipients_ = 1; // skip recipient rebuild on mail preview
           return qApi('Mailing', 'create', params).then(function(result) {
             mailing.modified_date = result.values[result.id].modified_date;
             // changes rolled back, so we don't care about updating mailing
@@ -290,11 +291,8 @@
       previewRecipients: function previewRecipients(mailing, previewLimit) {
         // To get list of recipients, we tentatively save the mailing and
         // get the resulting recipients -- then rollback any changes.
-        var params = angular.extend({}, mailing, mailing.recipients, {
-          name: 'placeholder', // for previewing recipients on new, incomplete mailing
-          subject: 'placeholder', // for previewing recipients on new, incomplete mailing
-          options: {force_rollback: 1},
-          'api.mailing_job.create': 1, // note: exact match to API default
+        var params = angular.extend({}, mailing.recipients, {
+          id: mailing.id,
           'api.MailingRecipients.get': {
             mailing_id: '$value.id',
             options: {limit: previewLimit},
@@ -317,10 +315,7 @@
           // To get list of recipients, we tentatively save the mailing and
           // get the resulting recipients -- then rollback any changes.
           var params = angular.extend({}, mailing, mailing.recipients, {
-            name: 'placeholder', // for previewing recipients on new, incomplete mailing
-            subject: 'placeholder', // for previewing recipients on new, incomplete mailing
-            options: {force_rollback: 1},
-            'api.mailing_job.create': 1, // note: exact match to API default
+            id: mailing.id,
             'api.MailingRecipients.getcount': {
               mailing_id: '$value.id'
             }
@@ -373,7 +368,7 @@
         delete params.jobs;
 
         delete params.recipients; // the content was merged in
-
+        params._skip_evil_bao_auto_recipients_ = 1; // skip recipient rebuild on simple save
         return qApi('Mailing', 'create', params).then(function(result) {
           if (result.id && !mailing.id) {
             mailing.id = result.id;
@@ -427,6 +422,8 @@
 
         delete params.recipients; // the content was merged in
 
+        params._skip_evil_bao_auto_recipients_ = 1; // skip recipient rebuild while sending test mail
+
         return qApi('Mailing', 'create', params).then(function (result) {
           if (result.id && !mailing.id) {
             mailing.id = result.id;

tests/phpunit/CiviTest/CiviUnitTestCase.php

@@ -1018,12 +1018,13 @@ public function organizationCreate($params = array(), $seq = 0) {
    *   parameters for civicrm_contact_add api function call
    * @param int $seq
    *   sequence number if creating multiple individuals
+   * @param bool $random
    *
    * @return int
    *   id of Individual created
    */
-  public function individualCreate($params = array(), $seq = 0) {
-    $params = array_merge($this->sampleContact('Individual', $seq), $params);
+  public function individualCreate($params = array(), $seq = 0, $random = FALSE) {
+    $params = array_merge($this->sampleContact('Individual', $seq, $random), $params);
     return $this->_contactCreate($params);
   }
 
@@ -1054,7 +1055,7 @@ public function householdCreate($params = array(), $seq = 0) {
    * @return array
    *   properties of sample contact (ie. $params for API call)
    */
-  public function sampleContact($contact_type, $seq = 0) {
+  public function sampleContact($contact_type, $seq = 0, $random = FALSE) {
     $samples = array(
       'Individual' => array(
         // The number of values in each list need to be coprime numbers to not have duplicates
@@ -1078,6 +1079,9 @@ public function sampleContact($contact_type, $seq = 0) {
     $params = array('contact_type' => $contact_type);
     foreach ($samples[$contact_type] as $key => $values) {
       $params[$key] = $values[$seq % count($values)];
+      if ($random) {
+        $params[$key] .= substr(sha1(rand()), 0, 5);
+      }
     }
     if ($contact_type == 'Individual') {
       $params['email'] = strtolower(
@@ -1870,13 +1874,14 @@ public function smartGroupCreate($smartGroupParams = array(), $groupParams = arr
    *
    * @param int $groupID
    * @param int $totalCount
+   * @param bool $random
    * @return int
    *    groupId of created group
    */
-  public function groupContactCreate($groupID, $totalCount = 10) {
+  public function groupContactCreate($groupID, $totalCount = 10, $random = FALSE) {
     $params = array('group_id' => $groupID);
     for ($i = 1; $i <= $totalCount; $i++) {
-      $contactID = $this->individualCreate();
+      $contactID = $this->individualCreate(array(), 0, $random);
       if ($i == 1) {
         $params += array('contact_id' => $contactID);
       }

tests/phpunit/api/v3/MailingABTest.php

@@ -121,8 +121,7 @@ public function groupPctProvider() {
    * @dataProvider groupPctProvider
    */
   public function testDistribution($totalGroupContacts, $groupPct, $expectedCountA, $expectedCountB, $expectedCountC) {
-
-    $result = $this->groupContactCreate($this->_groupID, $totalGroupContacts);
+    $result = $this->groupContactCreate($this->_groupID, $totalGroupContacts, TRUE);
     $this->assertEquals($totalGroupContacts, $result['added'], "in line " . __LINE__);
 
     $params = $this->_params;

tests/phpunit/api/v3/MailingTest.php

@@ -288,7 +288,6 @@ public function testMailerPreviewRecipients() {
     $params['mailings']['include'] = array();
     $params['mailings']['exclude'] = array();
     $params['options']['force_rollback'] = 1;
-    $params['api.mailing_job.create'] = 1;
     $params['api.MailingRecipients.get'] = array(
       'mailing_id' => '$value.id',
       'api.contact.getvalue' => array(
@@ -302,12 +301,10 @@ public function testMailerPreviewRecipients() {
 
     $maxIDs = array(
       'mailing' => CRM_Core_DAO::singleValueQuery('SELECT MAX(id) FROM civicrm_mailing'),
-      'job' => CRM_Core_DAO::singleValueQuery('SELECT MAX(id) FROM civicrm_mailing_job'),
       'group' => CRM_Core_DAO::singleValueQuery('SELECT MAX(id) FROM civicrm_mailing_group'),
     );
     $create = $this->callAPIAndDocument('Mailing', 'create', $params, __FUNCTION__, __FILE__);
     $this->assertDBQuery($maxIDs['mailing'], 'SELECT MAX(id) FROM civicrm_mailing'); // 'Preview should not create any mailing records'
-    $this->assertDBQuery($maxIDs['job'], 'SELECT MAX(id) FROM civicrm_mailing_job'); // 'Preview should not create any mailing_job record'
     $this->assertDBQuery($maxIDs['group'], 'SELECT MAX(id) FROM civicrm_mailing_group'); // 'Preview should not create any mailing_group records'
 
     $preview = $create['values'][$create['id']]['api.MailingRecipients.get'];
@@ -346,7 +343,6 @@ public function testMailerPreviewRecipientsDeduplicate() {
     $params['mailings']['include'] = array();
     $params['options']['force_rollback'] = 1;
     $params['dedupe_email'] = 1;
-    $params['api.mailing_job.create'] = 1;
     $params['api.MailingRecipients.get'] = array(
       'mailing_id' => '$value.id',
       'api.contact.getvalue' => array(