Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updates transform character scramble to return empty strings and nils properly #2692

Merged
merged 2 commits into from
Sep 13, 2024
Merged

updates transform character scramble to return empty strings and nils properly #2692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5210814
updates transform character scramble to return empty strings
nickzelei Sep 13, 2024
d4845c5
updates to handle nil
nickzelei Sep 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 18 additions & 19 deletions worker/pkg/benthos/transformers/transform_character_scramble.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package transformers

import (
"errors"
"fmt"
"regexp"
"strings"
Expand All @@ -28,16 +27,11 @@
Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs."))

err := bloblang.RegisterFunctionV2("transform_character_scramble", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) {
valuePtr, err := args.GetOptionalString("value")
value, err := args.GetOptionalString("value")
if err != nil {
return nil, err
}

var value string
if valuePtr != nil {
value = *valuePtr
}

regexPtr, err := args.GetOptionalString("user_provided_regex")
if err != nil {
return nil, err
Expand Down Expand Up @@ -80,17 +74,22 @@
return nil, fmt.Errorf("invalid parsed opts: %T", opts)
}

valueStr, ok := value.(string)
if !ok {
return nil, errors.New("value is not a string")
var strPtr *string
switch v := value.(type) {
case string:
strPtr = &v
case *string:
strPtr = v
default:
return nil, fmt.Errorf("transform_character_scramble: value is not string or *string, got %T", value)

Check warning on line 84 in worker/pkg/benthos/transformers/transform_character_scramble.go

View check run for this annotation

Codecov / codecov/patch

worker/pkg/benthos/transformers/transform_character_scramble.go#L77-L84 

Added lines #L77 - L84 were not covered by tests
}

regex := ""
if parsedOpts.userProvidedRegex != nil && *parsedOpts.userProvidedRegex != "" {
regex = *parsedOpts.userProvidedRegex
}

return transformCharacterScramble(parsedOpts.randomizer, valueStr, regex)
return transformCharacterScramble(parsedOpts.randomizer, strPtr, regex)

Check warning on line 92 in worker/pkg/benthos/transformers/transform_character_scramble.go

View check run for this annotation

Codecov / codecov/patch

worker/pkg/benthos/transformers/transform_character_scramble.go#L92 

Added line #L92 was not covered by tests
}

/*
Expand All @@ -104,9 +103,9 @@
Note that this does not work for hex values: 0x00 -> 0x1F
*/

func transformCharacterScramble(randomizer rng.Rand, value, regex string) (*string, error) {
if value == "" {
return nil, nil
func transformCharacterScramble(randomizer rng.Rand, value *string, regex string) (*string, error) {
if value == nil || *value == "" {
return value, nil
}

if regex != "" {
Expand All @@ -116,27 +115,27 @@
}

// finds all matches in a string
matches := reg.FindAllStringIndex(value, -1)
transformedString := value
matches := reg.FindAllStringIndex(*value, -1)

// if no matches are found just scramble the entire string
if matches == nil {
transformedString := strings.Map(randomizedScrambleChar(randomizer), value)
transformedString := strings.Map(randomizedScrambleChar(randomizer), *value)
return &transformedString, nil
}

// match is a [][]int with the inner []int being the start and end index values of the match
transformedString := *value
for _, match := range matches {
start, end := match[0], match[1]
// run the scrambler for the substring
matchTransformed := strings.Map(randomizedScrambleChar(randomizer), value[start:end])
matchTransformed := strings.Map(randomizedScrambleChar(randomizer), transformedString[start:end])
// replace the original substring with its transformed version
transformedString = transformedString[:start] + matchTransformed + transformedString[end:]
}

return &transformedString, nil
} else {
transformedString := strings.Map(randomizedScrambleChar(randomizer), value)
transformedString := strings.Map(randomizedScrambleChar(randomizer), *value)
return &transformedString, nil
}
}
Expand Down
52 changes: 34 additions & 18 deletions worker/pkg/benthos/transformers/transform_character_scramble_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils"
"github.com/nucleuscloud/neosync/worker/pkg/rng"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/warpstreamlabs/bento/public/bloblang"
)

Expand All @@ -32,7 +33,7 @@ func Test_ScrambleCharacter(t *testing.T) {
func Test_TransformCharacterSubstitutionLetters(t *testing.T) {
testStringValue := "he11o world"

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, "e11")
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, "e11")

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -43,7 +44,7 @@ func Test_TransformCharacterSubstitutionLetters(t *testing.T) {
func Test_TransformCharacterSubstitutionCapitalizationLetters(t *testing.T) {
testStringValue := "Hello"

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, helloWorldRegex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, helloWorldRegex)

assert.NoError(t, err)
assert.NotNil(t, res, "Result should not be nil")
Expand All @@ -57,7 +58,7 @@ func Test_TransformCharacterSubstitutionCapitalizationLetters(t *testing.T) {
func Test_TransformCharacterSubstitutionNumbers(t *testing.T) {
testStringValue := "41323421"

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, numberRegex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, numberRegex)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -68,7 +69,7 @@ func Test_TransformCharacterSubstitutionNumbers(t *testing.T) {
func Test_TransformCharacterSubstitutionLettersNumbers(t *testing.T) {
testStringValue := "hello wor23r2ld 221"

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, helloWorldRegex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, helloWorldRegex)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -79,7 +80,7 @@ func Test_TransformCharacterSubstitutionLettersNumbers(t *testing.T) {
func Test_TransformCharacterSubstitutionLettersNumbersCharacters(t *testing.T) {
testStringValue := "h#*(&lo wor23r2ld 221"

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, `#\*\(&`)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, `#\*\(&`)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -89,7 +90,7 @@ func Test_TransformCharacterSubstitutionLettersNumbersCharacters(t *testing.T) {

func Test_TransformCharacterSubstitutionLettersMultipleMatches(t *testing.T) {
// should match the first two sections and not that last i.e. h_ello_ello_elo
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), helloTest, `ello`)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &helloTest, `ello`)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -101,7 +102,7 @@ func Test_TransformCharacterSubstitutionLettersMultipleMatches(t *testing.T) {

func Test_TransformCharacterSubstitutionLettersNoMatches(t *testing.T) {
// should match the first two sections and not that last i.e. h_ello_ello_elo
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), helloTest, `123`)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &helloTest, `123`)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -112,7 +113,7 @@ func Test_TransformCharacterSubstitutionLettersNoMatches(t *testing.T) {

func Test_TransformCharacterSubstitutionLettersNilregex(t *testing.T) {
// should match the first two sections and not that last i.e. h_ello_ello_elo
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, ``)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, ``)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -126,7 +127,7 @@ func Test_TransformCharacterSubstitutionLettersMatchNumbers(t *testing.T) {
testStringValue := "MED-133-R123"
complexRegex := `\d+`

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, complexRegex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, complexRegex)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -152,7 +153,7 @@ func Test_TransformCharacterSubstitutionLettersSemiComplexRegex(t *testing.T) {
testStringValue := "MED-133-L123"
complexRegex := `-(.+?)-`

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, complexRegex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, complexRegex)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -167,7 +168,7 @@ func Test_TransformCharacterSubstitutionLettersComplexRegex(t *testing.T) {
testStringValue := "MED-133-A123"
complexRegex := `-(.+?)-`

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testStringValue, complexRegex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testStringValue, complexRegex)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand Down Expand Up @@ -209,7 +210,7 @@ func Test_TransformCharacterSubsitutitionRegexEmail(t *testing.T) {

testEmail := "nick@gmail.com"

res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), testEmail, emailregex)
res, err := transformCharacterScramble(rng.New(time.Now().UnixNano()), &testEmail, emailregex)

assert.NoError(t, err)
assert.IsType(t, "", *res)
Expand All @@ -218,12 +219,27 @@ func Test_TransformCharacterSubsitutitionRegexEmail(t *testing.T) {
assert.Equal(t, testEmail[:4], (*res)[:4], "The username should be the same")
}

func Test_TransformCharacterSubstitutionTransformerWithEmptyValue(t *testing.T) {
nilString := ""
mapping := fmt.Sprintf(`root = transform_character_scramble(value:%q,user_provided_regex:%q)`, nilString, nilString)
func Test_TransformCharacterSubstitutionTransformer_EmptyValue(t *testing.T) {
emptyString := ""
mapping := fmt.Sprintf(`root = transform_character_scramble(value:%q)`, emptyString)
ex, err := bloblang.Parse(mapping)
assert.NoError(t, err, "failed to parse the character substitution transformer")
require.NoError(t, err, "failed to parse the character substitution transformer")

_, err = ex.Query(nil)
assert.NoError(t, err)
res, err := ex.Query(nil)
require.NoError(t, err)
require.NotNil(t, res, "The response shouldnt be nil")

responseStr, ok := res.(*string)
require.True(t, ok)
require.NotNil(t, responseStr)
require.Equal(t, emptyString, *responseStr)
}

func Test_TransformCharacterSubstitutionTransformer_NilValue(t *testing.T) {
ex, err := bloblang.Parse("root = transform_character_scramble()")
require.NoError(t, err, "failed to parse the character substitution transformer")

res, err := ex.Query(nil)
require.NoError(t, err)
require.Nil(t, res, "The response was not nil")
}