Skip to content

Commit

Permalink
Unique now works on maps and arrays #2068
Browse files Browse the repository at this point in the history
  • Loading branch information
mikefarah committed Jun 15, 2024
1 parent 1147113 commit 42120e1
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 9 deletions.
46 changes: 45 additions & 1 deletion pkg/yqlib/doc/operators/unique.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,29 @@ will output
- ~
```
## Unique array object fields
## Unique array objects
Given a sample.yml file of:
```yaml
- name: harry
pet: cat
- name: billy
pet: dog
- name: harry
pet: cat
```
then
```bash
yq 'unique' sample.yml
```
will output
```yaml
- name: harry
pet: cat
- name: billy
pet: dog
```
## Unique array of objects by a field
Given a sample.yml file of:
```yaml
- name: harry
Expand All @@ -85,3 +107,25 @@ will output
pet: dog
```
## Unique array of arrays
Given a sample.yml file of:
```yaml
- - cat
- dog
- - cat
- sheep
- - cat
- dog
```
then
```bash
yq 'unique' sample.yml
```
will output
```yaml
- - cat
- dog
- - cat
- sheep
```
29 changes: 22 additions & 7 deletions pkg/yqlib/operator_unique.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
candidate := el.Value.(*CandidateNode)

if candidate.Kind != SequenceNode {
return Context{}, fmt.Errorf("Only arrays are supported for unique")
return Context{}, fmt.Errorf("only arrays are supported for unique")
}

var newMatches = orderedmap.NewOrderedMap()
Expand All @@ -34,12 +34,9 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
return Context{}, err
}

keyValue := "null"

if rhs.MatchingNodes.Len() > 0 {
first := rhs.MatchingNodes.Front()
keyCandidate := first.Value.(*CandidateNode)
keyValue = keyCandidate.Value
keyValue, err := getUniqueKeyValue(rhs)
if err != nil {
return Context{}, err
}

_, exists := newMatches.Get(keyValue)
Expand All @@ -59,3 +56,21 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
return context.ChildContext(results), nil

}

func getUniqueKeyValue(rhs Context) (string, error) {
keyValue := "null"

if rhs.MatchingNodes.Len() > 0 {
first := rhs.MatchingNodes.Front()
keyCandidate := first.Value.(*CandidateNode)
keyValue = keyCandidate.Value
if keyCandidate.Kind != ScalarNode {
var err error
keyValue, err = encodeToString(keyCandidate, encoderPreferences{YamlFormat, 0})
if err != nil {
return "", err
}
}
}
return keyValue, nil
}
18 changes: 17 additions & 1 deletion pkg/yqlib/operator_unique_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,29 @@ var uniqueOperatorScenarios = []expressionScenario{
},
},
{
description: "Unique array object fields",
description: "Unique array objects",
document: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: cat}]`,
expression: `unique`,
expected: []string{
"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n",
},
},
{
description: "Unique array of objects by a field",
document: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`,
expression: `unique_by(.name)`,
expected: []string{
"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n",
},
},
{
description: "Unique array of arrays",
document: `[[cat,dog], [cat, sheep], [cat,dog]]`,
expression: `unique`,
expected: []string{
"D0, P[], (!!seq)::[[cat, dog], [cat, sheep]]\n",
},
},
{
skipDoc: true,
document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
Expand Down

0 comments on commit 42120e1

Please sign in to comment.