Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generalize source key implementation to arbitrary-length keys #146

Merged
merged 2 commits into from
Aug 22, 2023
Merged

Generalize source key implementation to arbitrary-length keys #146

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3d77901
Generalize source key implementation to arbitrary-length keys
Aug 17, 2023
3042e85
Remove unneeded source key properties
Aug 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 54 additions & 79 deletions pkg/sourcekey/sourcekey.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package sourcekey

import (
"fmt"
"math"
"math/bits"
"sort"
"strings"
)

Expand All @@ -16,113 +19,85 @@ type SourceKey struct {
Namespace string

Class string

Parts []string
HappyTetrahedron marked this conversation as resolved.
Show resolved Hide resolved
HappyTetrahedron marked this conversation as resolved.
Show resolved Hide resolved
}

// Parse parses a source key in the format of "query:zone:tenant:namespace:class" or "query:zone:tenant:namespace".
func Parse(raw string) (SourceKey, error) {
parts := strings.Split(raw, elementSeparator)
if parts[len(parts)-1] == "" {
parts = parts[0 : len(parts)-1]
}
if len(parts) == 4 {
return SourceKey{parts[0], parts[1], parts[2], parts[3], ""}, nil
} else if len(parts) == 5 {
return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4]}, nil
return SourceKey{parts[0], parts[1], parts[2], parts[3], "", parts}, nil
} else if len(parts) >= 5 {
return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4], parts}, nil
}

return SourceKey{}, fmt.Errorf("expected key with 4 to 5 elements separated by `%s` got %d", elementSeparator, len(parts))
return SourceKey{}, fmt.Errorf("expected key with at least 4 elements separated by `%s` got %d", elementSeparator, len(parts))
}

// String returns the string representation "query:zone:tenant:namespace:class" of the key.
func (k SourceKey) String() string {
elements := []string{k.Query, k.Zone, k.Tenant, k.Namespace}
if k.Class != "" {
elements = append(elements, k.Class)
}
return strings.Join(elements, elementSeparator)
return strings.Join(k.Parts, elementSeparator)
}

// LookupKeys generates lookup keys for a dimension object in the database.
// The logic is described here: https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea
func (k SourceKey) LookupKeys() []string {
return generateSourceKeys(k.Query, k.Zone, k.Tenant, k.Namespace, k.Class)
}

func generateSourceKeys(query, zone, tenant, namespace, class string) []string {
keys := make([]string, 0)
base := []string{query, zone, tenant, namespace}
wildcardPositions := []int{1, 2}

if class != "" {
wildcardPositions = append(wildcardPositions, 3)
base = append(base, class)
}

for i := len(base); i > 0; i-- {
keys = append(keys, strings.Join(base[:i], elementSeparator))

for j := 1; j < len(wildcardPositions)+1; j++ {
perms := combinations(wildcardPositions, j)
for _, wcpos := range reverse(perms) {
elements := append([]string{}, base[:i]...)
for _, p := range wcpos {
elements[p] = "*"
currentKeyBase := k.Parts

for len(currentKeyBase) > 1 {
// For the base key of a given length l, the inner l-2 elements are to be replaced with wildcards in all possible combinations.
// To that end, generate 2^(l-2) binary numbers, sort them by specificity, and then for each number generate a key where
// for each 1-digit, the element is replaced with a wildcard (and for a 0-digit, the element is kept as-is).
innerLength := len(currentKeyBase) - 2
nums := makeRange(0, int(math.Pow(2, float64(innerLength))))
sort.Sort(sortBySpecificity(nums))
for i := range nums {
currentKeyElements := make([]string, 0)
currentKeyElements = append(currentKeyElements, currentKeyBase[0])
for digit := 0; digit < innerLength; digit++ {
if nums[i]&uint(math.Pow(2, float64(innerLength-1-digit))) > 0 {
currentKeyElements = append(currentKeyElements, "*")
} else {
currentKeyElements = append(currentKeyElements, currentKeyBase[1+digit])
}
keys = append(keys, strings.Join(elements, elementSeparator))
}
currentKeyElements = append(currentKeyElements, currentKeyBase[len(currentKeyBase)-1])
keys = append(keys, strings.Join(currentKeyElements, elementSeparator))
}
if i > 2 {
wildcardPositions = wildcardPositions[:len(wildcardPositions)-1]
}
currentKeyBase = currentKeyBase[0 : len(currentKeyBase)-1]
}

keys = append(keys, currentKeyBase[0])
return keys
}

func reverse(s [][]int) [][]int {
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
// SortBySpecificity sorts an array of uints representing binary numbers, such that numbers with fewer 1-digits come first.
// Numbers with an equal amount of 1-digits are sorted by magnitude.
type sortBySpecificity []uint

func (a sortBySpecificity) Len() int { return len(a) }
func (a sortBySpecificity) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a sortBySpecificity) Less(i, j int) bool {
onesI := bits.OnesCount(a[i])
onesJ := bits.OnesCount(a[j])
if onesI < onesJ {
return true
}
return s
}

func combinations(iterable []int, r int) (rt [][]int) {
pool := iterable
n := len(pool)

if r > n {
return
if onesI > onesJ {
return false
}
return a[i] < a[j]
}

indices := make([]int, r)
for i := range indices {
indices[i] = i
}

result := make([]int, r)
for i, el := range indices {
result[i] = pool[el]
}
s2 := make([]int, r)
copy(s2, result)
rt = append(rt, s2)

for {
i := r - 1
for ; i >= 0 && indices[i] == i+n-r; i -= 1 {
}

if i < 0 {
return
}

indices[i] += 1
for j := i + 1; j < r; j += 1 {
indices[j] = indices[j-1] + 1
}

for ; i < len(indices); i += 1 {
result[i] = pool[indices[i]]
}
s2 = make([]int, r)
copy(s2, result)
rt = append(rt, s2)
func makeRange(min, max int) []uint {
a := make([]uint, max-min)
for i := range a {
a[i] = uint(min + i)
}
return a
}
73 changes: 63 additions & 10 deletions pkg/sourcekey/sourcekey_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,35 +16,38 @@ func TestParseInvalidKey(t *testing.T) {
func TestParseWithClass(t *testing.T) {
k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd")
require.NoError(t, err)
require.Equal(t, k, sourcekey.SourceKey{
require.Equal(t, sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
})
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"},
}, k)
}

func TestParseWithoutClass(t *testing.T) {
k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234")
require.NoError(t, err)
require.Equal(t, k, sourcekey.SourceKey{
require.Equal(t, sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
})
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}, k)
}

func TestParseWithEmptyClass(t *testing.T) {
k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:")
require.NoError(t, err)
require.Equal(t, k, sourcekey.SourceKey{
require.Equal(t, sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
})
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}, k)
}

func TestStringWithClass(t *testing.T) {
Expand All @@ -54,6 +57,7 @@ func TestStringWithClass(t *testing.T) {
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"},
}
require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", key.String())
}
Expand All @@ -64,6 +68,7 @@ func TestStringWithoutClass(t *testing.T) {
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}
require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", key.String())
}
Expand All @@ -74,9 +79,10 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) {
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}.LookupKeys()

require.Equal(t, keys, []string{
require.Equal(t, []string{
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234",
Expand All @@ -85,7 +91,7 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) {
"appuio_cloud_storage:*:acme-corp",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2",
"appuio_cloud_storage",
})
}, keys)
}

func TestGenerateSourceKeysWithClass(t *testing.T) {
Expand All @@ -95,9 +101,10 @@ func TestGenerateSourceKeysWithClass(t *testing.T) {
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"},
}.LookupKeys()

require.Equal(t, keys, []string{
require.Equal(t, []string{
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd",
Expand All @@ -114,5 +121,51 @@ func TestGenerateSourceKeysWithClass(t *testing.T) {
"appuio_cloud_storage:*:acme-corp",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2",
"appuio_cloud_storage",
})
}, keys)
}

func TestGenerateSourceKeysWithSixElements(t *testing.T) {
keys := sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd", "exoscale"},
}.LookupKeys()

require.Equal(t, []string{
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd:exoscale",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:*:acme-corp:*:ssd:exoscale",
"appuio_cloud_storage:*:*:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:*:exoscale",
"appuio_cloud_storage:*:acme-corp:*:*:exoscale",
"appuio_cloud_storage:*:*:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:*:*:*:ssd:exoscale",
"appuio_cloud_storage:*:*:*:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd",
"appuio_cloud_storage:*:acme-corp:*:ssd",
"appuio_cloud_storage:*:*:sparkling-sound-1234:ssd",
"appuio_cloud_storage:*:*:*:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234",
"appuio_cloud_storage:*:*:sparkling-sound-1234",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp",
"appuio_cloud_storage:*:acme-corp",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2",
"appuio_cloud_storage",
}, keys)
}