APIv4 - Add UNIQUE flag to GROUP_CONCAT

Unlike DISTINCT which dedupes by the value of a field, UNIQUE will dedupe by the id of the record
civicrm · Aug 24, 2023 · 5fcccbf · 5fcccbf
1 parent d5a7597
commit 5fcccbf
Show file tree

Hide file tree

Showing 15 changed files with 109 additions and 30 deletions.
diff --git a/Civi/Api4/Query/Api4EntitySetQuery.php b/Civi/Api4/Query/Api4EntitySetQuery.php
@@ -119,7 +119,7 @@ protected function buildSelectClause() {
       $expr = SqlExpression::convert($item, TRUE);
       $alias = $expr->getAlias();
       $this->selectAliases[$alias] = $expr->getExpr();
-      $this->query->select($expr->render($this) . " AS `$alias`");
+      $this->query->select($expr->render($this, TRUE));
     }
   }
 

diff --git a/Civi/Api4/Query/Api4SelectQuery.php b/Civi/Api4/Query/Api4SelectQuery.php
@@ -198,7 +198,7 @@ protected function buildSelectClause($select = NULL) {
           throw new \CRM_Core_Exception('Cannot use existing field name as alias');
         }
         $this->selectAliases[$alias] = $expr->getExpr();
-        $this->query->select($expr->render($this) . " AS `$alias`");
+        $this->query->select($expr->render($this, TRUE));
       }
     }
   }

diff --git a/Civi/Api4/Query/SqlBool.php b/Civi/Api4/Query/SqlBool.php
@@ -21,8 +21,8 @@ class SqlBool extends SqlExpression {
   protected function initialize() {
   }
 
-  public function render(Api4Query $query): string {
-    return $this->expr === 'TRUE' ? '1' : '0';
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
+    return ($this->expr === 'TRUE' ? '1' : '0') . ($includeAlias ? " AS `{$this->getAlias()}`" : '');
   }
 
   public static function getTitle(): string {

diff --git a/Civi/Api4/Query/SqlEquation.php b/Civi/Api4/Query/SqlEquation.php
@@ -77,9 +77,10 @@ public function getArgs(): array {
    * Render the expression for insertion into the sql query
    *
    * @param \Civi\Api4\Query\Api4Query $query
+   * @param bool $includeAlias
    * @return string
    */
-  public function render(Api4Query $query): string {
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
     $output = [];
     foreach ($this->args as $i => $arg) {
       // Just an operator
@@ -98,7 +99,7 @@ public function render(Api4Query $query): string {
         $output[] = $arg->render($query);
       }
     }
-    return '(' . implode(' ', $output) . ')';
+    return '(' . implode(' ', $output) . ')' . ($includeAlias ? " AS `{$this->getAlias()}`" : '');
   }
 
   /**

diff --git a/Civi/Api4/Query/SqlExpression.php b/Civi/Api4/Query/SqlExpression.php
@@ -145,9 +145,12 @@ public function getFields(): array {
    * Renders expression to a sql string, replacing field names with column names.
    *
    * @param \Civi\Api4\Query\Api4Query $query
+   * @param bool $includeAlias
    * @return string
    */
-  abstract public function render(Api4Query $query): string;
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
+    return $this->expr . ($includeAlias ? " AS `{$this->getAlias()}`" : '');
+  }
 
   /**
    * @return string

diff --git a/Civi/Api4/Query/SqlField.php b/Civi/Api4/Query/SqlField.php
@@ -25,13 +25,13 @@ protected function initialize() {
     $this->fields[] = $this->expr;
   }
 
-  public function render(Api4Query $query): string {
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
     $field = $query->getField($this->expr, TRUE);
+    $rendered = $field['sql_name'];
     if (!empty($field['sql_renderer'])) {
-      $renderer = $field['sql_renderer'];
-      return $renderer($field, $query);
+      $rendered = $field['sql_renderer']($field, $query);
     }
-    return $field['sql_name'];
+    return $rendered . ($includeAlias ? " AS `{$this->getAlias()}`" : '');
   }
 
   public static function getTitle(): string {

diff --git a/Civi/Api4/Query/SqlFunction.php b/Civi/Api4/Query/SqlFunction.php
@@ -141,17 +141,18 @@ public function formatOutputValue(?string &$dataType, array &$values, string $ke
    * Render the expression for insertion into the sql query
    *
    * @param \Civi\Api4\Query\Api4Query $query
+   * @param bool $includeAlias
    * @return string
    */
-  public function render(Api4Query $query): string {
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
     $output = '';
     foreach ($this->args as $arg) {
       $rendered = $this->renderArg($arg, $query);
       if (strlen($rendered)) {
         $output .= (strlen($output) ? ' ' : '') . $rendered;
       }
     }
-    return $this->renderExpression($output);
+    return $this->renderExpression($output) . ($includeAlias ? " AS `{$this->getAlias()}`" : '');
   }
 
   /**
@@ -160,7 +161,7 @@ public function render(Api4Query $query): string {
    * @param string $output
    * @return string
    */
-  protected function renderExpression($output): string {
+  protected function renderExpression(string $output): string {
     return $this->getName() . '(' . $output . ')';
   }
 

diff --git a/Civi/Api4/Query/SqlFunctionDAYSTOANNIV.php b/Civi/Api4/Query/SqlFunctionDAYSTOANNIV.php
@@ -46,7 +46,7 @@ public static function getDescription(): string {
   /**
    * @inheritDoc
    */
-  protected function renderExpression($output): string {
+  protected function renderExpression(string $output): string {
     return "DATEDIFF(
       IF(
           DATE(CONCAT(YEAR(CURDATE()), '-', MONTH({$output}), '-', DAY({$output}))) < CURDATE(),

diff --git a/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php b/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php
@@ -11,6 +11,8 @@
 
 namespace Civi\Api4\Query;
 
+use Civi\Api4\Utils\CoreUtil;
+
 /**
  * Sql function
  */
@@ -23,7 +25,7 @@ class SqlFunctionGROUP_CONCAT extends SqlFunction {
   protected static function params(): array {
     return [
       [
-        'flag_before' => ['' => NULL, 'DISTINCT' => ts('Distinct')],
+        'flag_before' => ['' => NULL, 'DISTINCT' => ts('Distinct Value'), 'UNIQUE' => ts('Unique Record')],
         'max_expr' => 1,
         'must_be' => ['SqlField', 'SqlFunction', 'SqlEquation'],
         'optional' => FALSE,
@@ -68,6 +70,17 @@ public function formatOutputValue(?string &$dataType, array &$values, string $ke
           $exprArgs[0]['expr'][0]->formatOutputValue($dataType, $values[$key], $index);
         }
       }
+      // Perform deduping by unique id
+      if ($this->args[0]['prefix'] === ['UNIQUE'] && isset($values["_$key"])) {
+        $ids = \CRM_Utils_Array::explodePadded($values["_$key"]);
+        unset($values["_$key"]);
+        foreach ($ids as $index => $id) {
+          if (in_array($id, array_slice($ids, 0, $index))) {
+            unset($values[$key][$index]);
+          }
+        }
+        $values[$key] = array_values($values[$key]);
+      }
     }
     // If using custom separator, preserve raw string
     else {
@@ -89,4 +102,40 @@ public static function getDescription(): string {
     return ts('All values in the grouping.');
   }
 
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
+    $result = '';
+    // Handle pseudo-prefix `UNIQUE` which is like `DISTINCT` but based on the record id rather than the field value
+    if ($this->args[0]['prefix'] === ['UNIQUE']) {
+      $this->args[0]['prefix'] = [];
+      $expr = $this->args[0]['expr'][0];
+      $field = $query->getField($expr->getFields()[0]);
+      if ($field) {
+        $idField = CoreUtil::getIdFieldName($field['entity']);
+        $idFieldKey = substr($expr->getFields()[0], 0, 0 - strlen($field['name'])) . $idField;
+        // Keep the ordering consistent
+        if (empty($this->args[1]['prefix'])) {
+          $this->args[1] = [
+            'prefix' => ['ORDER BY'],
+            'expr' => [SqlExpression::convert($idFieldKey)],
+            'suffix' => [],
+          ];
+        }
+        // Already a unique field, so DISTINCT will work fine
+        if ($field['name'] === $idField) {
+          $this->args[0]['prefix'] = ['DISTINCT'];
+        }
+        // Add a unique field on which to dedupe in postprocessing (@see self::formatOutputValue)
+        elseif ($includeAlias) {
+          $orderByKey = $this->args[1]['expr'][0]->getFields()[0];
+          $extraSelectAlias = '_' . $this->getAlias();
+          $extraSelect = SqlExpression::convert("GROUP_CONCAT($idFieldKey ORDER BY $orderByKey) AS $extraSelectAlias", TRUE);
+          $query->selectAliases[$extraSelectAlias] = $extraSelect->getExpr();
+          $result .= $extraSelect->render($query, TRUE) . ',';
+        }
+      }
+    }
+    $result .= parent::render($query, $includeAlias);
+    return $result;
+  }
+
 }
diff --git a/Civi/Api4/Query/SqlNull.php b/Civi/Api4/Query/SqlNull.php
@@ -19,10 +19,6 @@ class SqlNull extends SqlExpression {
   protected function initialize() {
   }
 
-  public function render(Api4Query $query): string {
-    return 'NULL';
-  }
-
   public static function getTitle(): string {
     return ts('Null');
   }

diff --git a/Civi/Api4/Query/SqlNumber.php b/Civi/Api4/Query/SqlNumber.php
@@ -22,10 +22,6 @@ protected function initialize() {
     \CRM_Utils_Type::validate($this->expr, 'Float');
   }
 
-  public function render(Api4Query $query): string {
-    return $this->expr;
-  }
-
   public static function getTitle(): string {
     return ts('Number');
   }

diff --git a/Civi/Api4/Query/SqlString.php b/Civi/Api4/Query/SqlString.php
@@ -27,8 +27,8 @@ protected function initialize() {
     $this->expr = str_replace(['\\\\', "\\$quot", $backslash], [$backslash, $quot, '\\\\'], $str);
   }
 
-  public function render(Api4Query $query): string {
-    return '"' . \CRM_Core_DAO::escapeString($this->expr) . '"';
+  public function render(Api4Query $query, bool $includeAlias = FALSE): string {
+    return '"' . \CRM_Core_DAO::escapeString($this->expr) . '"' . ($includeAlias ? " AS `{$this->getAlias()}`" : '');
   }
 
   /**

diff --git a/Civi/Api4/Query/SqlWild.php b/Civi/Api4/Query/SqlWild.php
@@ -19,10 +19,6 @@ class SqlWild extends SqlExpression {
   protected function initialize() {
   }
 
-  public function render(Api4Query $query): string {
-    return '*';
-  }
-
   public static function getTitle(): string {
     return ts('Wild');
   }

diff --git a/Civi/Api4/Utils/FormattingUtil.php b/Civi/Api4/Utils/FormattingUtil.php
@@ -232,6 +232,10 @@ public static function formatOutputValues(&$result, $fields, $action = 'get', $s
       }
     }
     foreach ($result as $key => $value) {
+      // Skip null values or values that have already been unset by `formatOutputValue` functions
+      if (!isset($result[$key])) {
+        continue;
+      }
       $fieldExpr = SqlExpression::convert($selectAliases[$key] ?? $key);
       $fieldName = \CRM_Utils_Array::first($fieldExpr->getFields() ?? '');
       $baseName = $fieldName ? \CRM_Utils_Array::first(explode(':', $fieldName)) : NULL;

diff --git a/tests/phpunit/api/v4/Action/SqlFunctionTest.php b/tests/phpunit/api/v4/Action/SqlFunctionTest.php
@@ -106,6 +106,39 @@ public function testGroupAggregates() {
     $this->assertEquals(['January', 'February', 'March', 'April'], $agg['months']);
   }
 
+  public function testGroupConcatUnique(): void {
+    $cid1 = $this->createTestRecord('Contact')['id'];
+    $cid2 = $this->createTestRecord('Contact')['id'];
+
+    $this->saveTestRecords('Address', [
+      'records' => [
+        ['contact_id' => $cid1, 'city' => 'A', 'location_type_id' => 1],
+        ['contact_id' => $cid1, 'city' => 'A', 'location_type_id' => 2],
+        ['contact_id' => $cid1, 'city' => 'B', 'location_type_id' => 3],
+      ],
+    ]);
+    $this->saveTestRecords('Email', [
+      'records' => [
+        ['contact_id' => $cid1, 'email' => 'test1@example.org', 'location_type_id' => 1],
+        ['contact_id' => $cid1, 'email' => 'test2@example.org', 'location_type_id' => 2],
+      ],
+    ]);
+
+    $result = Contact::get(FALSE)
+      ->addSelect('GROUP_CONCAT(UNIQUE address.id) AS address_id')
+      ->addSelect('GROUP_CONCAT(UNIQUE address.city) AS address_city')
+      ->addSelect('GROUP_CONCAT(UNIQUE email.email) AS email')
+      ->addGroupBy('id')
+      ->addJoin('Address AS address', 'LEFT', ['id', '=', 'address.contact_id'])
+      ->addJoin('Email AS email', 'LEFT', ['id', '=', 'email.contact_id'])
+      ->addOrderBy('id')
+      ->addWhere('id', 'IN', [$cid1, $cid2])
+      ->execute();
+
+    $this->assertEquals(['A', 'A', 'B'], $result[0]['address_city']);
+    $this->assertEquals(['test1@example.org', 'test2@example.org'], $result[0]['email']);
+  }
+
   public function testGroupHaving() {
     $cid = Contact::create(FALSE)->addValue('first_name', 'donor')->execute()->first()['id'];
     Contribution::save(FALSE)