Skip to content

Commit

Permalink
Add "ModifyQuery" to CRUD interface for arbitrary SQL processing
Browse files Browse the repository at this point in the history 
While ReadQueryModifier allows CRUD models to have static additions
(for joins or complex filtering), this will allow for more dynamic
additions to any query.

Signed-off-by: Andrew Richardson <andrew.richardson@kaleido.io>
  • Loading branch information
@awrichar
awrichar committed Oct 27, 2023
1 parent 6f614c3 commit 1ab3a18
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 20 deletions.
42 changes: 31 additions & 11 deletions pkg/dbsql/crud.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,25 +91,30 @@ func (r *ResourceBase) SetUpdated(t *fftypes.FFTime) {
r.Updated = t
}

type CRUD[T Resource] interface {
Validate()
Upsert(ctx context.Context, inst T, optimization UpsertOptimization, hooks ...PostCompletionHook) (created bool, err error)
InsertMany(ctx context.Context, instances []T, allowPartialSuccess bool, hooks ...PostCompletionHook) (err error)
Insert(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error)
Replace(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error)
type CRUDQuery[T Resource] interface {
GetByID(ctx context.Context, id string, getOpts ...GetOption) (inst T, err error)
GetByUUIDOrName(ctx context.Context, uuidOrName string, getOpts ...GetOption) (result T, err error)
GetByName(ctx context.Context, name string, getOpts ...GetOption) (instance T, err error)
GetFirst(ctx context.Context, filter ffapi.Filter, getOpts ...GetOption) (instance T, err error)
GetSequenceForID(ctx context.Context, id string) (seq int64, err error)
GetMany(ctx context.Context, filter ffapi.Filter) (instances []T, fr *ffapi.FilterResult, err error)
Count(ctx context.Context, filter ffapi.Filter) (count int64, err error)
ModifyQuery(modifier QueryModifier) CRUDQuery[T]
}

type CRUD[T Resource] interface {
CRUDQuery[T]
Validate()
Upsert(ctx context.Context, inst T, optimization UpsertOptimization, hooks ...PostCompletionHook) (created bool, err error)
InsertMany(ctx context.Context, instances []T, allowPartialSuccess bool, hooks ...PostCompletionHook) (err error)
Insert(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error)
Replace(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error)
Update(ctx context.Context, id string, update ffapi.Update, hooks ...PostCompletionHook) (err error)
UpdateSparse(ctx context.Context, sparseUpdate T, hooks ...PostCompletionHook) (err error)
UpdateMany(ctx context.Context, filter ffapi.Filter, update ffapi.Update, hooks ...PostCompletionHook) (err error)
Delete(ctx context.Context, id string, hooks ...PostCompletionHook) (err error)
DeleteMany(ctx context.Context, filter ffapi.Filter, hooks ...PostCompletionHook) (err error) // no events
Scoped(scope sq.Eq) *CrudBase[T] // allows dynamic scoping to a collection
Scoped(scope sq.Eq) CRUD[T] // allows dynamic scoping to a collection
}

type CrudBase[T Resource] struct {
Expand All @@ -133,15 +138,30 @@ type CrudBase[T Resource] struct {
// Optional extensions
ReadTableAlias string
ReadOnlyColumns []string
ReadQueryModifier func(sq.SelectBuilder) sq.SelectBuilder
ReadQueryModifier QueryModifier
}

func (c *CrudBase[T]) Scoped(scope sq.Eq) *CrudBase[T] {
func (c *CrudBase[T]) Scoped(scope sq.Eq) CRUD[T] {
cScoped := *c
cScoped.ScopedFilter = func() sq.Eq { return scope }
return &cScoped
}

func (c *CrudBase[T]) ModifyQuery(newModifier QueryModifier) CRUDQuery[T] {
cModified := *c
originalModifier := cModified.ReadQueryModifier
cModified.ReadQueryModifier = func(sb sq.SelectBuilder) sq.SelectBuilder {
if originalModifier != nil {
sb = originalModifier(sb)
}
if newModifier != nil {
sb = newModifier(sb)
}
return sb
}
return &cModified
}

func UUIDValidator(ctx context.Context, idStr string) error {
_, err := fftypes.ParseUUID(ctx, idStr)
return err
Expand Down Expand Up @@ -656,7 +676,7 @@ func (c *CrudBase[T]) getManyScoped(ctx context.Context, tableFrom string, fi *f
}
instances = append(instances, inst)
}
return instances, c.DB.QueryRes(ctx, c.Table, tx, fop, fi), err
return instances, c.DB.QueryRes(ctx, c.Table, tx, fop, c.ReadQueryModifier, fi), err
}

func (c *CrudBase[T]) Count(ctx context.Context, filter ffapi.Filter) (count int64, err error) {
Expand All @@ -675,7 +695,7 @@ func (c *CrudBase[T]) Count(ctx context.Context, filter ffapi.Filter) (count int
fop,
}
}
return c.DB.CountQuery(ctx, c.Table, nil, fop, "*")
return c.DB.CountQuery(ctx, c.Table, nil, fop, c.ReadQueryModifier, "*")
}

func (c *CrudBase[T]) Update(ctx context.Context, id string, update ffapi.Update, hooks ...PostCompletionHook) (err error) {
Expand Down
10 changes: 10 additions & 0 deletions pkg/dbsql/crud_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,16 @@ func TestCRUDWithDBEnd2End(t *testing.T) {
assert.NoError(t, err)
checkEqualExceptTimes(t, *c1, *c1copy)

// Check we get it back with custom modifiers
collection.ReadQueryModifier = func(sb sq.SelectBuilder) sq.SelectBuilder {
return sb.Where(sq.Eq{"ns": "ns1"})
}
c1copy, err = iCrud.ModifyQuery(func(sb sq.SelectBuilder) sq.SelectBuilder {
return sb.Where(sq.Eq{"field1": "hello1"})
}).GetByName(ctx, *c1.Name)
assert.NoError(t, err)
checkEqualExceptTimes(t, *c1, *c1copy)

// Upsert the existing row optimized
c1copy.Field1 = strPtr("hello again - 1")
created, err := iCrud.Upsert(ctx, c1copy, UpsertOptimizationExisting)
Expand Down
11 changes: 8 additions & 3 deletions pkg/dbsql/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ type Database struct {
sequenceColumn string
}

type QueryModifier = func(sq.SelectBuilder) sq.SelectBuilder

// PreCommitAccumulator is a structure that can accumulate state during
// the transaction, then has a function that is called just before commit.
type PreCommitAccumulator interface {
Expand Down Expand Up @@ -221,7 +223,7 @@ func (s *Database) Query(ctx context.Context, table string, q sq.SelectBuilder)
return s.QueryTx(ctx, table, nil, q)
}

func (s *Database) CountQuery(ctx context.Context, table string, tx *TXWrapper, fop sq.Sqlizer, countExpr string) (count int64, err error) {
func (s *Database) CountQuery(ctx context.Context, table string, tx *TXWrapper, fop sq.Sqlizer, qm QueryModifier, countExpr string) (count int64, err error) {
count = -1
l := log.L(ctx)
if tx == nil {
Expand All @@ -233,6 +235,9 @@ func (s *Database) CountQuery(ctx context.Context, table string, tx *TXWrapper,
countExpr = "*"
}
q := sq.Select(fmt.Sprintf("COUNT(%s)", countExpr)).From(table).Where(fop)
if qm != nil {
q = qm(q)
}
sqlQuery, args, err := q.PlaceholderFormat(s.features.PlaceholderFormat).ToSql()
if err != nil {
return count, i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed)
Expand All @@ -259,10 +264,10 @@ func (s *Database) CountQuery(ctx context.Context, table string, tx *TXWrapper,
return count, nil
}

func (s *Database) QueryRes(ctx context.Context, table string, tx *TXWrapper, fop sq.Sqlizer, fi *ffapi.FilterInfo) *ffapi.FilterResult {
func (s *Database) QueryRes(ctx context.Context, table string, tx *TXWrapper, fop sq.Sqlizer, qm QueryModifier, fi *ffapi.FilterInfo) *ffapi.FilterResult {
fr := &ffapi.FilterResult{}
if fi.Count {
count, err := s.CountQuery(ctx, table, tx, fop, fi.CountExpr)
count, err := s.CountQuery(ctx, table, tx, fop, qm, fi.CountExpr)
if err != nil {
// Log, but continue
log.L(ctx).Warnf("Unable to return count for query: %s", err)
Expand Down
21 changes: 16 additions & 5 deletions pkg/dbsql/database_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,14 +499,14 @@ func TestRollbackFail(t *testing.T) {

func TestCountQueryBadSQL(t *testing.T) {
s, _ := NewMockProvider().UTInit()
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Insert("wrong"), "")
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Insert("wrong"), nil, "")
assert.Regexp(t, "FF00174", err)
}

func TestCountQueryQueryFailed(t *testing.T) {
s, mdb := NewMockProvider().UTInit()
mdb.ExpectQuery("^SELECT COUNT\\(\\*\\)").WillReturnError(fmt.Errorf("pop"))
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Eq{"col1": "val1"}, "")
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Eq{"col1": "val1"}, nil, "")
assert.Regexp(t, "FF00176.*pop", err)
}

Expand All @@ -516,21 +516,32 @@ func TestCountQueryScanFailTx(t *testing.T) {
mdb.ExpectQuery("^SELECT COUNT\\(\\*\\)").WillReturnRows(sqlmock.NewRows([]string{"col1"}).AddRow("not a number"))
ctx, tx, _, err := s.BeginOrUseTx(context.Background())
assert.NoError(t, err)
_, err = s.CountQuery(ctx, "table1", tx, sq.Eq{"col1": "val1"}, "")
_, err = s.CountQuery(ctx, "table1", tx, sq.Eq{"col1": "val1"}, nil, "")
assert.Regexp(t, "FF00182", err)
}

func TestCountQueryWithExpr(t *testing.T) {
s, mdb := NewMockProvider().UTInit()
mdb.ExpectQuery("^SELECT COUNT\\(DISTINCT key\\)").WillReturnRows(sqlmock.NewRows([]string{"col1"}).AddRow(10))
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Eq{"col1": "val1"}, "DISTINCT key")
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Eq{"col1": "val1"}, nil, "DISTINCT key")
assert.NoError(t, err)
assert.NoError(t, mdb.ExpectationsWereMet())
}

func TestCountQueryWithModifier(t *testing.T) {
s, mdb := NewMockProvider().UTInit()
mdb.ExpectQuery("^SELECT COUNT\\(\\*\\)").WillReturnRows(sqlmock.NewRows([]string{"col1"}).AddRow(10))
qm := func(sb sq.SelectBuilder) sq.SelectBuilder {
return sb.Where(sq.Eq{"col1": "val1"})
}
_, err := s.CountQuery(context.Background(), "table1", nil, sq.Eq{"col1": "val1"}, qm, "")
assert.NoError(t, err)
assert.NoError(t, mdb.ExpectationsWereMet())
}

func TestQueryResSwallowError(t *testing.T) {
s, _ := NewMockProvider().UTInit()
res := s.QueryRes(context.Background(), "table1", nil, sq.Insert("wrong"), &ffapi.FilterInfo{
res := s.QueryRes(context.Background(), "table1", nil, sq.Insert("wrong"), nil, &ffapi.FilterInfo{
Count: true,
})
assert.Equal(t, int64(-1), *res.TotalCount)
Expand Down
4 changes: 3 additions & 1 deletion pkg/dbsql/filter_sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,9 @@ func (s *Database) filterSelectFinalized(ctx context.Context, tableName string,
sort[i] = fmt.Sprintf("%s%s%s", s.mapFieldName(tableName, sf.Field, typeMap), direction, nulls)
}
sortString = strings.Join(sort, ", ")
sel = sel.OrderBy(sortString)
if sortString != "" {
sel = sel.OrderBy(sortString)
}
if fi.Skip > 0 {
sel = sel.Offset(fi.Skip)
}
Expand Down

0 comments on commit 1ab3a18

Please sign in to comment.