Merge pull request #8511 from jitendrapurohit/CRM-18439

CRM-18439 - support MySQL 5.7 sql_mode=only_full_group_by

CRM/Activity/BAO/Activity.php

@@ -717,7 +717,7 @@ public static function &getActivities($input) {
     $insertSQL = "INSERT INTO {$activityTempTable} (" . implode(',', $insertValueSQL) . " ) ";
 
     $order = $limit = $groupBy = '';
-    $groupBy = " GROUP BY tbl.activity_id ";
+    $groupBy = " GROUP BY tbl.activity_id, tbl.activity_type, tbl.case_id, tbl.case_subject ";
 
     if (!empty($input['sort'])) {
       if (is_a($input['sort'], 'CRM_Utils_Sort')) {
@@ -804,7 +804,7 @@ public static function &getActivities($input) {
 INNER JOIN civicrm_activity_contact ac ON ( ac.activity_id = {$activityTempTable}.activity_id )
 INNER JOIN civicrm_contact c ON c.id = ac.contact_id
 WHERE ac.record_type_id = %1
-GROUP BY ac.activity_id
+GROUP BY ac.activity_id, ac.contact_id
 ";
 
     CRM_Core_DAO::executeQuery($query, $params);

CRM/Campaign/BAO/Survey.php

@@ -479,8 +479,7 @@ public static function voterDetails($voterIds, $returnProperties = array()) {
       $query = "
   SELECT  contact.id as contactId, $selectClause
     FROM  $fromClause
-   WHERE  $whereClause
-Group By  contact.id";
+   WHERE  $whereClause";
 
       $contact = CRM_Core_DAO::executeQuery($query);
       while ($contact->fetch()) {

CRM/Campaign/Form/Survey/Results.php

@@ -61,7 +61,7 @@ public function preProcess() {
       $this->set('values', $this->_values);
     }
 
-    $query = "SELECT MAX(id) as id, title FROM civicrm_report_instance WHERE name = %1";
+    $query = "SELECT MAX(id) as id, title FROM civicrm_report_instance WHERE name = %1 GROUP BY id";
     $params = array(1 => array("survey_{$this->_surveyId}", 'String'));
     $result = CRM_Core_DAO::executeQuery($query, $params);
     if ($result->fetch()) {

CRM/Case/BAO/Case.php

@@ -458,7 +458,6 @@ public static function getCaseActivityQuery($type = 'upcoming', $userID = NULL,
       FROM civicrm_view_case_activity_upcoming
       ORDER BY activity_date_time ASC, id ASC
       ) AS upcomingOrdered
-    GROUP BY case_id
     ) AS act
   LEFT JOIN civicrm_option_group aog ON aog.name='activity_type'
   LEFT JOIN civicrm_option_value aov ON ( aov.option_group_id = aog.id AND aov.value = act.activity_type_id )
@@ -478,7 +477,6 @@ public static function getCaseActivityQuery($type = 'upcoming', $userID = NULL,
       FROM civicrm_view_case_activity_recent
       ORDER BY activity_date_time DESC, id ASC
       ) AS recentOrdered
-    GROUP BY case_id
     ) AS act
 LEFT JOIN civicrm_option_group aog ON aog.name='activity_type'
   LEFT JOIN civicrm_option_value aov ON ( aov.option_group_id = aog.id AND aov.value = act.activity_type_id )
@@ -759,6 +757,7 @@ public static function getCasesSummary($allCases = TRUE, $userID) {
       $myCaseWhereClause = " AND case_relationship.contact_id_b = {$userID}";
       $myGroupByClause = " GROUP BY CONCAT(case_relationship.case_id,'-',case_relationship.contact_id_b)";
     }
+    $myGroupByClause .= ", case_status.label, status_id, case_type_id";
 
     // FIXME: This query could be a lot more efficient if it used COUNT() instead of returning all rows and then counting them with php
     $query = "
@@ -986,7 +985,7 @@ public static function getCaseActivity($caseID, &$params, $contactID, $context =
     }
 
     $groupBy = "
-         GROUP BY ca.id ";
+         GROUP BY ca.id, tcc.id, scc.id, acc.id, ov.value";
 
     $sortBy = CRM_Utils_Array::value('sortBy', $params);
     if (!$sortBy) {
@@ -1246,8 +1245,7 @@ public static function getRelatedContacts($caseID, $skipDetails = FALSE) {
       LEFT JOIN civicrm_email ce
         ON ce.contact_id = cc.id
         AND ce.is_primary= 1
-      WHERE cr.case_id =  %1
-      GROUP BY cc.id';
+      WHERE cr.case_id =  %1';
 
     $params = array(1 => array($caseID, 'Integer'));
     $dao = CRM_Core_DAO::executeQuery($query, $params);
@@ -1732,7 +1730,7 @@ public static function getCaseActivityDates($caseID, $criteriaParams = array(),
       if (!empty($criteriaParams['activity_type_id'])) {
         $where .= " AND ca.activity_type_id    = " . CRM_Utils_Type::escape($criteriaParams['activity_type_id'], 'Integer');
         $where .= " AND ca.is_current_revision = 1";
-        $groupBy .= " GROUP BY ca.activity_type_id";
+        $groupBy .= " GROUP BY ca.activity_type_id, ca.id";
       }
 
       if (!empty($criteriaParams['newest'])) {

CRM/Contact/BAO/Contact/Utils.php

@@ -777,12 +777,13 @@ public static function contactDetails($componentIds, $componentName, $returnProp
       $fromClause = implode(' ', $from);
       $selectClause = implode(', ', $select);
       $whereClause = "{$compTable}.id IN (" . implode(',', $componentIds) . ')';
+      $groupBy = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($select, array("{$compTable}.id", 'contact.id'));
 
       $query = "
   SELECT  contact.id as contactId, $compTable.id as componentId, $selectClause
     FROM  $compTable as $compTable $fromClause
    WHERE  $whereClause
-Group By  componentId";
+   {$groupBy}";
 
       $contact = CRM_Core_DAO::executeQuery($query);
       while ($contact->fetch()) {

CRM/Contact/BAO/Group.php

@@ -854,7 +854,6 @@ public static function getGroupList(&$params) {
           ON createdBy.id = groups.created_id
         {$from}
         WHERE $whereClause {$where}
-        GROUP BY groups.id
         {$orderBy}
         {$limit}";
 

CRM/Contact/BAO/GroupContact.php

@@ -280,7 +280,7 @@ public static function getGroupList($contactId = 0, $visibility = FALSE) {
 
     $select = $from = $where = '';
 
-    $select = 'SELECT DISTINCT civicrm_group.id, civicrm_group.title ';
+    $select = 'SELECT civicrm_group.id, civicrm_group.title ';
     $from = ' FROM civicrm_group ';
     $where = " WHERE civicrm_group.is_active = 1 ";
     if ($contactId) {
@@ -292,9 +292,10 @@ public static function getGroupList($contactId = 0, $visibility = FALSE) {
     if ($visibility) {
       $where .= " AND civicrm_group.visibility != 'User and User Admin Only'";
     }
+    $groupBy = " GROUP BY civicrm_group.id";
 
     $orderby = " ORDER BY civicrm_group.name";
-    $sql = $select . $from . $where . $orderby;
+    $sql = $select . $from . $where . $groupBy . $orderby;
 
     $group->query($sql);
 

CRM/Contact/BAO/Query.php

@@ -4334,7 +4334,7 @@ public static function apiQuery(
 
     // add group by
     if ($query->_useGroupBy) {
-      $sql .= ' GROUP BY contact_a.id';
+      $sql .= self::getGroupByFromSelectColumns($query->_select, 'contact_a.id');
     }
     if (!empty($sort)) {
       $sort = CRM_Utils_Type::escape($sort, 'String');
@@ -4520,6 +4520,59 @@ public static function filterCountryFromValuesIfStateExists(&$formValues) {
     }
   }
 
+  /**
+   * Include Select columns in groupBy clause.
+   *
+   * @param array $selectClauses
+   * @param array $groupBy - Columns already included in GROUP By clause.
+   *
+   * @return string
+   */
+  public static function getGroupByFromSelectColumns($selectClauses, $groupBy = NULL) {
+    $groupBy = (array) $groupBy;
+    $mysqlVersion = CRM_Core_DAO::singleValueQuery('SELECT VERSION()');
+    $sqlMode = CRM_Core_DAO::singleValueQuery('SELECT @@sql_mode');
+
+    //return if ONLY_FULL_GROUP_BY is not enabled.
+    if (!version_compare($mysqlVersion, '5.7', '<') && !empty($sqlMode) && in_array('ONLY_FULL_GROUP_BY', explode(',', $sqlMode))) {
+      $regexToExclude = '/(ROUND|AVG|COUNT|GROUP_CONCAT|SUM|MAX|MIN)\(/i';
+      foreach ($selectClauses as $key => $val) {
+        $aliasArray = preg_split('/ as /i', $val);
+        // if more than 1 alias we need to split by ','.
+        if (count($aliasArray) > 2) {
+          $aliasArray = preg_split('/,/', $val);
+          foreach ($aliasArray as $key => $value) {
+            $alias = current(preg_split('/ as /i', $value));
+            if (!in_array($alias, $groupBy) && preg_match($regexToExclude, trim($alias)) !== 1) {
+              $groupBy[] = $alias;
+            }
+          }
+        }
+        else {
+          list($selectColumn, $alias) = array_pad($aliasArray, 2, NULL);
+          $dateRegex = '/^(DATE_FORMAT|DATE_ADD|CASE)/i';
+          $tableName = current(explode('.', $selectColumn));
+          $primaryKey = "{$tableName}.id";
+          // exclude columns which are already included in groupBy and aggregate functions from select
+          // CRM-18439 - Also exclude the columns which are functionally dependent on columns in $groupBy (MySQL 5.7+)
+          if (!in_array($selectColumn, $groupBy) && !in_array($primaryKey, $groupBy) && preg_match($regexToExclude, trim($selectColumn)) !== 1) {
+            if (!empty($alias) && preg_match($dateRegex, trim($selectColumn))) {
+              $groupBy[] = $alias;
+            }
+            else {
+              $groupBy[] = $selectColumn;
+            }
+          }
+        }
+      }
+    }
+
+    if (!empty($groupBy)) {
+      return " GROUP BY " . implode(', ', $groupBy);
+    }
+    return '';
+  }
+
   /**
    * Create and query the db for an contact search.
    *
@@ -4580,16 +4633,20 @@ public function searchQuery(
     }
 
     // building the query string
-    $groupBy = NULL;
+    $groupBy = $groupByCol = NULL;
     if (!$count) {
       if (isset($this->_groupByComponentClause)) {
         $groupBy = $this->_groupByComponentClause;
+        $groupCols = preg_replace('/^GROUP BY /', '', trim($this->_groupByComponentClause));
+        $groupByCol = explode(', ', $groupCols);
       }
       elseif ($this->_useGroupBy) {
+        $groupByCol = 'contact_a.id';
         $groupBy = ' GROUP BY contact_a.id';
       }
     }
     if ($this->_mode & CRM_Contact_BAO_Query::MODE_ACTIVITY && (!$count)) {
+      $groupByCol = 'civicrm_activity.id';
       $groupBy = 'GROUP BY civicrm_activity.id ';
     }
 
@@ -4612,6 +4669,10 @@ public function searchQuery(
 
     list($select, $from, $where, $having) = $this->query($count, $sortByChar, $groupContacts, $onlyDeleted);
 
+    if (!empty($groupByCol)) {
+      $groupBy = self::getGroupByFromSelectColumns($this->_select, $groupByCol);
+    }
+
     if ($additionalWhereClause) {
       $where = $where . ' AND ' . $additionalWhereClause;
     }
@@ -4669,7 +4730,8 @@ public function getCachedContacts($cacheKey, $offset, $rowCount, $includeContact
     list($select, $from, $where) = $this->query(FALSE, FALSE, FALSE, $onlyDeleted);
     $from = " FROM civicrm_prevnext_cache pnc INNER JOIN civicrm_contact contact_a ON contact_a.id = pnc.entity_id1 AND pnc.cacheKey = '$cacheKey' " . substr($from, 31);
     $order = " ORDER BY pnc.id";
-    $groupBy = " GROUP BY contact_a.id";
+    $groupByCol = array('contact_a.id', 'pnc.id');
+    $groupBy = self::getGroupByFromSelectColumns($this->_select, $groupByCol);
     $limit = " LIMIT $offset, $rowCount";
     $query = "$select $from $where $groupBy $order $limit";
 
@@ -4749,7 +4811,6 @@ public function &summaryContribution($context = NULL) {
 SELECT COUNT( conts.total_amount ) as total_count,
        SUM(   conts.total_amount ) as total_amount,
        AVG(   conts.total_amount ) as total_avg,
-       conts.total_amount          as amount,
        conts.currency              as currency";
     if ($this->_permissionWhereClause) {
       $where .= " AND " . $this->_permissionWhereClause;
@@ -4793,9 +4854,12 @@ public function &summaryContribution($context = NULL) {
 
     $orderBy = 'ORDER BY civicrm_contribution_total_amount_count DESC';
     $groupBy = 'GROUP BY currency, civicrm_contribution.total_amount';
-    $modeSQL = "$select, conts.civicrm_contribution_total_amount_count as civicrm_contribution_total_amount_count FROM ($innerQuery
-    $groupBy $orderBy) as conts
-    GROUP BY currency";
+    $modeSQL = "$select, SUBSTRING_INDEX(GROUP_CONCAT(conts.total_amount
+      ORDER BY conts.civicrm_contribution_total_amount_count DESC SEPARATOR ';'), ';', 1) as amount,
+      MAX(conts.civicrm_contribution_total_amount_count) as civicrm_contribution_total_amount_count
+      FROM ($innerQuery
+      $groupBy $orderBy) as conts
+      GROUP BY currency";
 
     $summary['total']['mode'] = CRM_Contribute_BAO_Contribution::computeStats('mode', $modeSQL);
 

CRM/Contact/Form/Search/Custom/EventAggregate.php

@@ -159,10 +159,11 @@ public function all(
     }
 
     $where = $this->where();
+    $groupFromSelect = "civicrm_option_value.label, civicrm_contribution.payment_instrument_id";
 
-    $groupBy = "event_id";
+    $groupBy = "event_id, event_type_id, {$groupFromSelect}";
     if (!empty($this->_formValues['event_type_id'])) {
-      $groupBy = "event_type_id";
+      $groupBy = "event_type_id, event_id, {$groupFromSelect}";
     }
 
     $sql = "

CRM/Contact/Form/Search/Custom/FullText/Activity.php

@@ -148,7 +148,6 @@ public function moveIDs($fromTable, $toTable, $limit) {
 LEFT JOIN  civicrm_case_activity cca ON cca.activity_id = ca.id
 LEFT JOIN  civicrm_case_contact ccc ON ccc.case_id = cca.case_id
 WHERE (ca.is_deleted = 0 OR ca.is_deleted IS NULL)
-GROUP BY ca.id
 {$this->toLimit($limit)}
 ";
     CRM_Core_DAO::executeQuery($sql);

CRM/Contribute/BAO/Contribution.php

@@ -4005,6 +4005,7 @@ public static function getPaymentInfo($id, $component, $getTrxnInfo = FALSE, $us
     if ($getTrxnInfo && $baseTrxnId) {
       $arRelationTypeId = key(CRM_Core_PseudoConstant::accountOptionValues('account_relationship', NULL, " AND v.name LIKE 'Accounts Receivable Account is' "));
       $arAccount = CRM_Contribute_PseudoConstant::financialAccountType($financialTypeId, $arRelationTypeId);
+
       // Need to exclude fee trxn rows so filter out rows where TO FINANCIAL ACCOUNT is expense account
       $sql = "
         SELECT GROUP_CONCAT(fa.`name`) as financial_account,
@@ -4021,8 +4022,8 @@ public static function getPaymentInfo($id, $component, $getTrxnInfo = FALSE, $us
           INNER JOIN civicrm_financial_account fa ON fa.id = fi.financial_account_id
 
         WHERE con.id = %1 AND ft.to_financial_account_id <> %3
-        GROUP BY ft.id
-      ";
+        GROUP BY ft.id";
+
       $queryParams = array(
         1 => array($contributionId, 'Integer'),
         2 => array($feeFinancialAccount, 'Integer'),

CRM/Contribute/BAO/Query.php

@@ -825,7 +825,7 @@ public static function isSoftCreditOptionEnabled($queryParams = array()) {
                SELECT con.id as id, con.contact_id, cso.id as filter_id, NULL as scredit_id
                  FROM civicrm_contribution con
             LEFT JOIN civicrm_contribution_soft cso ON con.id = cso.contribution_id
-             GROUP BY id, contact_id, scredit_id
+             GROUP BY id, contact_id, scredit_id, cso.id
             UNION ALL
                SELECT scredit.contribution_id as id, scredit.contact_id, scredit.id as filter_id, scredit.id as scredit_id
                  FROM civicrm_contribution_soft as scredit";

CRM/Contribute/Page/ContributionPage.php

@@ -675,7 +675,7 @@ public function pagerAtoZ($whereClause, $whereParams) {
 SELECT DISTINCT UPPER(LEFT(title, 1)) as sort_name
 FROM civicrm_contribution_page
 WHERE $whereClause
-ORDER BY LEFT(title, 1)
+ORDER BY UPPER(LEFT(title, 1))
 ";
     $dao = CRM_Core_DAO::executeQuery($query, $whereParams);
 

CRM/Event/BAO/Event.php

@@ -416,7 +416,6 @@ public static function getEventSummary() {
            ( civicrm_event.is_template IS NULL OR civicrm_event.is_template = 0) AND
            civicrm_event.start_date >= DATE_SUB( NOW(), INTERVAL 7 day )
            $validEventIDs
-GROUP BY   civicrm_event.id
 ORDER BY   civicrm_event.start_date ASC
 $event_summary_limit
 ";

CRM/Event/BAO/Participant.php

@@ -1931,7 +1931,7 @@ public static function changeFeeSelections($params, $participantId, $contributio
   SELECT fi.*, SUM(fi.amount) as differenceAmt, price_field_value_id, financial_type_id, tax_amount
     FROM civicrm_financial_item fi LEFT JOIN civicrm_line_item li ON (li.id = fi.entity_id AND fi.entity_table = 'civicrm_line_item')
 WHERE (li.entity_table = 'civicrm_participant' AND li.entity_id = {$participantId})
-GROUP BY li.entity_table, li.entity_id, price_field_value_id
+GROUP BY li.entity_table, li.entity_id, price_field_value_id, fi.id
 ";
       $updateFinancialItemInfoDAO = CRM_Core_DAO::executeQuery($updateFinancialItem);
       $trxn = CRM_Core_BAO_FinancialTrxn::getFinancialTrxnId($contributionId, 'DESC', TRUE);

CRM/Event/Page/ManageEvent.php

@@ -554,7 +554,7 @@ public function pagerAtoZ($whereClause, $whereParams) {
    SELECT DISTINCT UPPER(LEFT(title, 1)) as sort_name
      FROM civicrm_event
     WHERE $whereClause
- ORDER BY LEFT(title, 1)
+ ORDER BY UPPER(LEFT(title, 1))
 ";
     $dao = CRM_Core_DAO::executeQuery($query, $whereParams);