diff --git a/Civi/Api4/Query/Api4EntitySetQuery.php b/Civi/Api4/Query/Api4EntitySetQuery.php index 5e553f2217c9..3be5f7c2ce9b 100644 --- a/Civi/Api4/Query/Api4EntitySetQuery.php +++ b/Civi/Api4/Query/Api4EntitySetQuery.php @@ -119,7 +119,7 @@ protected function buildSelectClause() { $expr = SqlExpression::convert($item, TRUE); $alias = $expr->getAlias(); $this->selectAliases[$alias] = $expr->getExpr(); - $this->query->select($expr->render($this) . " AS `$alias`"); + $this->query->select($expr->render($this, TRUE)); } } diff --git a/Civi/Api4/Query/Api4SelectQuery.php b/Civi/Api4/Query/Api4SelectQuery.php index 26d09e1897af..40fec2708379 100644 --- a/Civi/Api4/Query/Api4SelectQuery.php +++ b/Civi/Api4/Query/Api4SelectQuery.php @@ -198,7 +198,7 @@ protected function buildSelectClause($select = NULL) { throw new \CRM_Core_Exception('Cannot use existing field name as alias'); } $this->selectAliases[$alias] = $expr->getExpr(); - $this->query->select($expr->render($this) . " AS `$alias`"); + $this->query->select($expr->render($this, TRUE)); } } } diff --git a/Civi/Api4/Query/SqlBool.php b/Civi/Api4/Query/SqlBool.php index 36aeb3ee1ff2..3577a31eb1b4 100644 --- a/Civi/Api4/Query/SqlBool.php +++ b/Civi/Api4/Query/SqlBool.php @@ -21,8 +21,8 @@ class SqlBool extends SqlExpression { protected function initialize() { } - public function render(Api4Query $query): string { - return $this->expr === 'TRUE' ? '1' : '0'; + public function render(Api4Query $query, bool $includeAlias = FALSE): string { + return ($this->expr === 'TRUE' ? '1' : '0') . ($includeAlias ? " AS `{$this->getAlias()}`" : ''); } public static function getTitle(): string { diff --git a/Civi/Api4/Query/SqlEquation.php b/Civi/Api4/Query/SqlEquation.php index ac0fb8cf2271..04f08cbe1568 100644 --- a/Civi/Api4/Query/SqlEquation.php +++ b/Civi/Api4/Query/SqlEquation.php @@ -77,9 +77,10 @@ public function getArgs(): array { * Render the expression for insertion into the sql query * * @param \Civi\Api4\Query\Api4Query $query + * @param bool $includeAlias * @return string */ - public function render(Api4Query $query): string { + public function render(Api4Query $query, bool $includeAlias = FALSE): string { $output = []; foreach ($this->args as $i => $arg) { // Just an operator @@ -98,7 +99,7 @@ public function render(Api4Query $query): string { $output[] = $arg->render($query); } } - return '(' . implode(' ', $output) . ')'; + return '(' . implode(' ', $output) . ')' . ($includeAlias ? " AS `{$this->getAlias()}`" : ''); } /** diff --git a/Civi/Api4/Query/SqlExpression.php b/Civi/Api4/Query/SqlExpression.php index 9416a9aa3a2f..9555152db6fa 100644 --- a/Civi/Api4/Query/SqlExpression.php +++ b/Civi/Api4/Query/SqlExpression.php @@ -145,9 +145,12 @@ public function getFields(): array { * Renders expression to a sql string, replacing field names with column names. * * @param \Civi\Api4\Query\Api4Query $query + * @param bool $includeAlias * @return string */ - abstract public function render(Api4Query $query): string; + public function render(Api4Query $query, bool $includeAlias = FALSE): string { + return $this->expr . ($includeAlias ? " AS `{$this->getAlias()}`" : ''); + } /** * @return string diff --git a/Civi/Api4/Query/SqlField.php b/Civi/Api4/Query/SqlField.php index 862f67aea087..0f2fb26aba20 100644 --- a/Civi/Api4/Query/SqlField.php +++ b/Civi/Api4/Query/SqlField.php @@ -25,13 +25,13 @@ protected function initialize() { $this->fields[] = $this->expr; } - public function render(Api4Query $query): string { + public function render(Api4Query $query, bool $includeAlias = FALSE): string { $field = $query->getField($this->expr, TRUE); + $rendered = $field['sql_name']; if (!empty($field['sql_renderer'])) { - $renderer = $field['sql_renderer']; - return $renderer($field, $query); + $rendered = $field['sql_renderer']($field, $query); } - return $field['sql_name']; + return $rendered . ($includeAlias ? " AS `{$this->getAlias()}`" : ''); } public static function getTitle(): string { diff --git a/Civi/Api4/Query/SqlFunction.php b/Civi/Api4/Query/SqlFunction.php index e2b8cb9118e0..44a0b097d595 100644 --- a/Civi/Api4/Query/SqlFunction.php +++ b/Civi/Api4/Query/SqlFunction.php @@ -141,9 +141,10 @@ public function formatOutputValue(?string &$dataType, array &$values, string $ke * Render the expression for insertion into the sql query * * @param \Civi\Api4\Query\Api4Query $query + * @param bool $includeAlias * @return string */ - public function render(Api4Query $query): string { + public function render(Api4Query $query, bool $includeAlias = FALSE): string { $output = ''; foreach ($this->args as $arg) { $rendered = $this->renderArg($arg, $query); @@ -151,7 +152,7 @@ public function render(Api4Query $query): string { $output .= (strlen($output) ? ' ' : '') . $rendered; } } - return $this->renderExpression($output); + return $this->renderExpression($output) . ($includeAlias ? " AS `{$this->getAlias()}`" : ''); } /** @@ -160,7 +161,7 @@ public function render(Api4Query $query): string { * @param string $output * @return string */ - protected function renderExpression($output): string { + protected function renderExpression(string $output): string { return $this->getName() . '(' . $output . ')'; } diff --git a/Civi/Api4/Query/SqlFunctionDAYSTOANNIV.php b/Civi/Api4/Query/SqlFunctionDAYSTOANNIV.php index b47a4a1b31aa..14f85b2ccc61 100644 --- a/Civi/Api4/Query/SqlFunctionDAYSTOANNIV.php +++ b/Civi/Api4/Query/SqlFunctionDAYSTOANNIV.php @@ -46,7 +46,7 @@ public static function getDescription(): string { /** * @inheritDoc */ - protected function renderExpression($output): string { + protected function renderExpression(string $output): string { return "DATEDIFF( IF( DATE(CONCAT(YEAR(CURDATE()), '-', MONTH({$output}), '-', DAY({$output}))) < CURDATE(), diff --git a/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php b/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php index dfb255a273d1..7af9a2f64d78 100644 --- a/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php +++ b/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php @@ -11,6 +11,8 @@ namespace Civi\Api4\Query; +use Civi\Api4\Utils\CoreUtil; + /** * Sql function */ @@ -23,7 +25,7 @@ class SqlFunctionGROUP_CONCAT extends SqlFunction { protected static function params(): array { return [ [ - 'flag_before' => ['' => NULL, 'DISTINCT' => ts('Distinct')], + 'flag_before' => ['' => NULL, 'DISTINCT' => ts('Distinct Value'), 'UNIQUE' => ts('Unique Record')], 'max_expr' => 1, 'must_be' => ['SqlField', 'SqlFunction', 'SqlEquation'], 'optional' => FALSE, @@ -68,6 +70,17 @@ public function formatOutputValue(?string &$dataType, array &$values, string $ke $exprArgs[0]['expr'][0]->formatOutputValue($dataType, $values[$key], $index); } } + // Perform deduping by unique id + if ($this->args[0]['prefix'] === ['UNIQUE'] && isset($values["_$key"])) { + $ids = \CRM_Utils_Array::explodePadded($values["_$key"]); + unset($values["_$key"]); + foreach ($ids as $index => $id) { + if (in_array($id, array_slice($ids, 0, $index))) { + unset($values[$key][$index]); + } + } + $values[$key] = array_values($values[$key]); + } } // If using custom separator, preserve raw string else { @@ -89,4 +102,40 @@ public static function getDescription(): string { return ts('All values in the grouping.'); } + public function render(Api4Query $query, bool $includeAlias = FALSE): string { + $result = ''; + // Handle pseudo-prefix `UNIQUE` which is like `DISTINCT` but based on the record id rather than the field value + if ($this->args[0]['prefix'] === ['UNIQUE']) { + $this->args[0]['prefix'] = []; + $expr = $this->args[0]['expr'][0]; + $field = $query->getField($expr->getFields()[0]); + if ($field) { + $idField = CoreUtil::getIdFieldName($field['entity']); + $idFieldKey = substr($expr->getFields()[0], 0, 0 - strlen($field['name'])) . $idField; + // Keep the ordering consistent + if (empty($this->args[1]['prefix'])) { + $this->args[1] = [ + 'prefix' => ['ORDER BY'], + 'expr' => [SqlExpression::convert($idFieldKey)], + 'suffix' => [], + ]; + } + // Already a unique field, so DISTINCT will work fine + if ($field['name'] === $idField) { + $this->args[0]['prefix'] = ['DISTINCT']; + } + // Add a unique field on which to dedupe in postprocessing (@see self::formatOutputValue) + elseif ($includeAlias) { + $orderByKey = $this->args[1]['expr'][0]->getFields()[0]; + $extraSelectAlias = '_' . $this->getAlias(); + $extraSelect = SqlExpression::convert("GROUP_CONCAT($idFieldKey ORDER BY $orderByKey) AS $extraSelectAlias", TRUE); + $query->selectAliases[$extraSelectAlias] = $extraSelect->getExpr(); + $result .= $extraSelect->render($query, TRUE) . ','; + } + } + } + $result .= parent::render($query, $includeAlias); + return $result; + } + } diff --git a/Civi/Api4/Query/SqlNull.php b/Civi/Api4/Query/SqlNull.php index 312f01fa4b55..935ff6cdf226 100644 --- a/Civi/Api4/Query/SqlNull.php +++ b/Civi/Api4/Query/SqlNull.php @@ -19,10 +19,6 @@ class SqlNull extends SqlExpression { protected function initialize() { } - public function render(Api4Query $query): string { - return 'NULL'; - } - public static function getTitle(): string { return ts('Null'); } diff --git a/Civi/Api4/Query/SqlNumber.php b/Civi/Api4/Query/SqlNumber.php index 82c784a845d2..d8e301f53c29 100644 --- a/Civi/Api4/Query/SqlNumber.php +++ b/Civi/Api4/Query/SqlNumber.php @@ -22,10 +22,6 @@ protected function initialize() { \CRM_Utils_Type::validate($this->expr, 'Float'); } - public function render(Api4Query $query): string { - return $this->expr; - } - public static function getTitle(): string { return ts('Number'); } diff --git a/Civi/Api4/Query/SqlString.php b/Civi/Api4/Query/SqlString.php index 1efcaf15bb36..cd6db545e524 100644 --- a/Civi/Api4/Query/SqlString.php +++ b/Civi/Api4/Query/SqlString.php @@ -27,8 +27,8 @@ protected function initialize() { $this->expr = str_replace(['\\\\', "\\$quot", $backslash], [$backslash, $quot, '\\\\'], $str); } - public function render(Api4Query $query): string { - return '"' . \CRM_Core_DAO::escapeString($this->expr) . '"'; + public function render(Api4Query $query, bool $includeAlias = FALSE): string { + return '"' . \CRM_Core_DAO::escapeString($this->expr) . '"' . ($includeAlias ? " AS `{$this->getAlias()}`" : ''); } /** diff --git a/Civi/Api4/Query/SqlWild.php b/Civi/Api4/Query/SqlWild.php index 972c47f1d8d6..f59bcf12082f 100644 --- a/Civi/Api4/Query/SqlWild.php +++ b/Civi/Api4/Query/SqlWild.php @@ -19,10 +19,6 @@ class SqlWild extends SqlExpression { protected function initialize() { } - public function render(Api4Query $query): string { - return '*'; - } - public static function getTitle(): string { return ts('Wild'); } diff --git a/Civi/Api4/Utils/FormattingUtil.php b/Civi/Api4/Utils/FormattingUtil.php index 252b71e84132..4a63e4db4192 100644 --- a/Civi/Api4/Utils/FormattingUtil.php +++ b/Civi/Api4/Utils/FormattingUtil.php @@ -232,6 +232,10 @@ public static function formatOutputValues(&$result, $fields, $action = 'get', $s } } foreach ($result as $key => $value) { + // Skip null values or values that have already been unset by `formatOutputValue` functions + if (!isset($result[$key])) { + continue; + } $fieldExpr = SqlExpression::convert($selectAliases[$key] ?? $key); $fieldName = \CRM_Utils_Array::first($fieldExpr->getFields() ?? ''); $baseName = $fieldName ? \CRM_Utils_Array::first(explode(':', $fieldName)) : NULL; diff --git a/tests/phpunit/api/v4/Action/SqlFunctionTest.php b/tests/phpunit/api/v4/Action/SqlFunctionTest.php index ca1003cb36a2..4b1b57b96067 100644 --- a/tests/phpunit/api/v4/Action/SqlFunctionTest.php +++ b/tests/phpunit/api/v4/Action/SqlFunctionTest.php @@ -106,6 +106,39 @@ public function testGroupAggregates() { $this->assertEquals(['January', 'February', 'March', 'April'], $agg['months']); } + public function testGroupConcatUnique(): void { + $cid1 = $this->createTestRecord('Contact')['id']; + $cid2 = $this->createTestRecord('Contact')['id']; + + $this->saveTestRecords('Address', [ + 'records' => [ + ['contact_id' => $cid1, 'city' => 'A', 'location_type_id' => 1], + ['contact_id' => $cid1, 'city' => 'A', 'location_type_id' => 2], + ['contact_id' => $cid1, 'city' => 'B', 'location_type_id' => 3], + ], + ]); + $this->saveTestRecords('Email', [ + 'records' => [ + ['contact_id' => $cid1, 'email' => 'test1@example.org', 'location_type_id' => 1], + ['contact_id' => $cid1, 'email' => 'test2@example.org', 'location_type_id' => 2], + ], + ]); + + $result = Contact::get(FALSE) + ->addSelect('GROUP_CONCAT(UNIQUE address.id) AS address_id') + ->addSelect('GROUP_CONCAT(UNIQUE address.city) AS address_city') + ->addSelect('GROUP_CONCAT(UNIQUE email.email) AS email') + ->addGroupBy('id') + ->addJoin('Address AS address', 'LEFT', ['id', '=', 'address.contact_id']) + ->addJoin('Email AS email', 'LEFT', ['id', '=', 'email.contact_id']) + ->addOrderBy('id') + ->addWhere('id', 'IN', [$cid1, $cid2]) + ->execute(); + + $this->assertEquals(['A', 'A', 'B'], $result[0]['address_city']); + $this->assertEquals(['test1@example.org', 'test2@example.org'], $result[0]['email']); + } + public function testGroupHaving() { $cid = Contact::create(FALSE)->addValue('first_name', 'donor')->execute()->first()['id']; Contribution::save(FALSE)