Skip to content

Commit

Permalink
feat: Query readings without pagination offset
Browse files Browse the repository at this point in the history 
Query readings without pagination offset to prevent large dataset query and count.

Signed-off-by: bruce <weichou1229@gmail.com>
(cherry picked from commit 1a89671151ac64faeeb06f70b556460da8f76616)
  • Loading branch information
@weichou1229
weichou1229 committed Jan 17, 2025
1 parent 19dbf70 commit ccee771
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 62 deletions.
138 changes: 84 additions & 54 deletions internal/core/data/application/reading.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,16 @@ func ReadingTotalCount(dic *di.Container) (uint32, errors.EdgeX) {
// AllReadings query events by offset, and limit
func AllReadings(offset int, limit int, dic *di.Container) (readings []dtos.BaseReading, totalCount uint32, err errors.EdgeX) {
dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.AllReadings(offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingTotalCount()
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -46,12 +56,6 @@ func AllReadings(offset int, limit int, dic *di.Container) (readings []dtos.Base
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.AllReadings(offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

Expand All @@ -62,6 +66,15 @@ func ReadingsByResourceName(offset int, limit int, resourceName string, dic *di.
}
dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.ReadingsByResourceName(offset, limit, resourceName)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingCountByResourceName(resourceName)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -70,12 +83,6 @@ func ReadingsByResourceName(offset int, limit int, resourceName string, dic *di.
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.ReadingsByResourceName(offset, limit, resourceName)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

Expand All @@ -86,6 +93,15 @@ func ReadingsByDeviceName(offset int, limit int, name string, dic *di.Container)
}
dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.ReadingsByDeviceName(offset, limit, name)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingCountByDeviceName(name)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -94,18 +110,22 @@ func ReadingsByDeviceName(offset int, limit int, name string, dic *di.Container)
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.ReadingsByDeviceName(offset, limit, name)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

// ReadingsByTimeRange query readings with offset, limit and time range
func ReadingsByTimeRange(start int64, end int64, offset int, limit int, dic *di.Container) (readings []dtos.BaseReading, totalCount uint32, err errors.EdgeX) {
dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.ReadingsByTimeRange(start, end, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingCountByTimeRange(start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -114,12 +134,6 @@ func ReadingsByTimeRange(start int64, end int64, offset int, limit int, dic *di.
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.ReadingsByTimeRange(start, end, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

Expand Down Expand Up @@ -151,6 +165,16 @@ func ReadingsByResourceNameAndTimeRange(resourceName string, start int64, end in
return readings, totalCount, errors.NewCommonEdgeX(errors.KindContractInvalid, "resourceName is empty", nil)
}
dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.ReadingsByResourceNameAndTimeRange(resourceName, start, end, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingCountByResourceNameAndTimeRange(resourceName, start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -159,12 +183,6 @@ func ReadingsByResourceNameAndTimeRange(resourceName string, start int64, end in
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.ReadingsByResourceNameAndTimeRange(resourceName, start, end, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

Expand All @@ -178,6 +196,16 @@ func ReadingsByDeviceNameAndResourceName(deviceName string, resourceName string,
}

dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.ReadingsByDeviceNameAndResourceName(deviceName, resourceName, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceName(deviceName, resourceName)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -186,12 +214,6 @@ func ReadingsByDeviceNameAndResourceName(deviceName string, resourceName string,
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.ReadingsByDeviceNameAndResourceName(deviceName, resourceName, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

Expand All @@ -205,6 +227,16 @@ func ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName string, resource
}

dbClient := container.DBClientFrom(dic.Get)

readingModels, err := dbClient.ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName, resourceName, start, end, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)

if offset < 0 {
return readings, 0, err // skip total count
}
totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceNameAndTimeRange(deviceName, resourceName, start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
Expand All @@ -213,12 +245,6 @@ func ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName string, resource
if !cont {
return []dtos.BaseReading{}, totalCount, err
}

readingModels, err := dbClient.ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName, resourceName, start, end, offset, limit)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
readings, err = convertReadingModelsToDTOs(readingModels)
return readings, totalCount, err
}

Expand All @@ -231,21 +257,25 @@ func ReadingsByDeviceNameAndResourceNamesAndTimeRange(deviceName string, resourc
dbClient := container.DBClientFrom(dic.Get)
var readingModels []models.Reading
if len(resourceNames) > 0 {
totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceNamesAndTimeRange(deviceName, resourceNames, start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont {
return []dtos.BaseReading{}, totalCount, err
if offset >= 0 {
totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceNamesAndTimeRange(deviceName, resourceNames, start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont {
return []dtos.BaseReading{}, totalCount, err
}
}
readingModels, err = dbClient.ReadingsByDeviceNameAndResourceNamesAndTimeRange(deviceName, resourceNames, start, end, offset, limit)
} else {
totalCount, err = dbClient.ReadingCountByDeviceNameAndTimeRange(deviceName, start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont {
return []dtos.BaseReading{}, totalCount, err
if offset >= 0 {
totalCount, err = dbClient.ReadingCountByDeviceNameAndTimeRange(deviceName, start, end)
if err != nil {
return readings, totalCount, errors.NewCommonEdgeXWrapper(err)
}
if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont {
return []dtos.BaseReading{}, totalCount, err
}
}
readingModels, err = dbClient.ReadingsByDeviceNameAndTimeRange(deviceName, start, end, offset, limit)
}
Expand Down
18 changes: 10 additions & 8 deletions internal/core/data/controller/http/reading.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
"github.com/labstack/echo/v4"
)

const minOffset = -1 // allow using -1 to query reading data and skip the total count for pagination

type ReadingController struct {
reader io.DtoReader
dic *di.Container
Expand Down Expand Up @@ -65,7 +67,7 @@ func (rc *ReadingController) AllReadings(c echo.Context) error {
config := dataContainer.ConfigurationFrom(rc.dic.Get)

// parse URL query string for offset, and limit, and labels
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand All @@ -87,7 +89,7 @@ func (rc *ReadingController) ReadingsByTimeRange(c echo.Context) error {
config := dataContainer.ConfigurationFrom(rc.dic.Get)

// parse time range (start, end), offset, and limit from incoming request
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand All @@ -111,7 +113,7 @@ func (rc *ReadingController) ReadingsByResourceName(c echo.Context) error {
resourceName := c.Param(common.ResourceName)

// parse URL query string for offset, limit
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand All @@ -135,7 +137,7 @@ func (rc *ReadingController) ReadingsByDeviceName(c echo.Context) error {
name := c.Param(common.Name)

// parse URL query string for offset, limit
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand Down Expand Up @@ -181,7 +183,7 @@ func (rc *ReadingController) ReadingsByResourceNameAndTimeRange(c echo.Context)
resourceName := c.Param(common.ResourceName)

// parse time range (start, end), offset, and limit from incoming request
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand All @@ -206,7 +208,7 @@ func (rc *ReadingController) ReadingsByDeviceNameAndResourceName(c echo.Context)
resourceName := c.Param(common.ResourceName)

// parse URL query string for offset, limit
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand All @@ -231,7 +233,7 @@ func (rc *ReadingController) ReadingsByDeviceNameAndResourceNameAndTimeRange(c e
resourceName := c.Param(common.ResourceName)

// parse time range (start, end), offset, and limit from incoming request
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand All @@ -256,7 +258,7 @@ func (rc *ReadingController) ReadingsByDeviceNameAndResourceNamesAndTimeRange(c
deviceName := c.Param(common.Name)

// parse time range (start, end), offset, and limit from incoming request
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
if err != nil {
return utils.WriteErrorResponse(w, ctx, lc, err, "")
}
Expand Down

0 comments on commit ccee771

Please sign in to comment.