From d4997e8de16f9fce28ed34b400a620e80f4533a9 Mon Sep 17 00:00:00 2001 From: Ramanan Ravikumar Date: Tue, 26 Sep 2023 16:21:43 +0530 Subject: [PATCH 1/2] Pagination for user audit logs api deepfence/enterprise-roadmap#1958 --- deepfence_server/apiDocs/operation.go | 4 ++-- deepfence_server/handler/audit_log.go | 13 ++++++++++++- deepfence_server/model/setting.go | 4 ++++ deepfence_server/router/router.go | 2 +- deepfence_utils/postgresql/postgresql-db/db.go | 2 +- deepfence_utils/postgresql/postgresql-db/models.go | 2 +- .../postgresql/postgresql-db/queries.sql.go | 14 ++++++++++---- deepfence_utils/postgresql/queries.sql | 3 ++- 8 files changed, 33 insertions(+), 11 deletions(-) diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index 4409d54c2a..5a9e5abbaf 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -735,9 +735,9 @@ func (d *OpenApiDocs) AddSettingsOperations() { d.AddOperation("updateSetting", http.MethodPatch, "/deepfence/settings/global-settings/{id}", "Update setting", "Update setting", http.StatusNoContent, []string{tagSettings}, bearerToken, new(SettingUpdateRequest), nil) - d.AddOperation("getUserActivityLogs", http.MethodGet, "/deepfence/settings/user-activity-log", + d.AddOperation("getUserActivityLogs", http.MethodPost, "/deepfence/settings/user-activity-log", "Get activity logs", "Get activity logs for all users", - http.StatusOK, []string{tagSettings}, bearerToken, nil, new([]postgresqldb.GetAuditLogsRow)) + http.StatusOK, []string{tagSettings}, bearerToken, new(GetAuditLogsRequest), new([]postgresqldb.GetAuditLogsRow)) // Scheduled tasks d.AddOperation("getScheduledTasks", http.MethodGet, "/deepfence/scheduled-task", diff --git a/deepfence_server/handler/audit_log.go b/deepfence_server/handler/audit_log.go index db340f6f50..75a1081534 100644 --- a/deepfence_server/handler/audit_log.go +++ b/deepfence_server/handler/audit_log.go @@ -156,6 +156,14 @@ func (h *Handler) AddAuditLog(namespace string, params postgresql_db.CreateAudit } func (h *Handler) GetAuditLogs(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var req model.GetAuditLogsRequest + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + h.respondError(&BadDecoding{err}, w) + return + } + ctx := r.Context() pgClient, err := directory.PostgresClient(ctx) if err != nil { @@ -164,7 +172,10 @@ func (h *Handler) GetAuditLogs(w http.ResponseWriter, r *http.Request) { return } - auditLogs, err := pgClient.GetAuditLogs(ctx) + auditLogs, err := pgClient.GetAuditLogs(ctx, postgresql_db.GetAuditLogsParams{ + Offset: int32(req.Window.Offset), + Limit: int32(req.Window.Size), + }) if err != nil { log.Error().Msgf("%v", err) h.respondError(err, w) diff --git a/deepfence_server/model/setting.go b/deepfence_server/model/setting.go index a62bfdf125..942c36db56 100644 --- a/deepfence_server/model/setting.go +++ b/deepfence_server/model/setting.go @@ -58,6 +58,10 @@ type SettingsResponse struct { Description string `json:"description" required:"true"` } +type GetAuditLogsRequest struct { + Window FetchWindow `json:"window" required:"true"` +} + type SettingUpdateRequest struct { ID int64 `path:"id" validate:"required" required:"true"` Key string `json:"key" validate:"required,oneof=console_url inactive_delete_scan_results" required:"true" enum:"console_url,inactive_delete_scan_results"` diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index 38861803c5..03b3980dab 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -206,7 +206,7 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c }) r.Route("/settings", func(r chi.Router) { - r.Get("/user-activity-log", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetAuditLogs)) + r.Post("/user-activity-log", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetAuditLogs)) r.Route("/global-settings", func(r chi.Router) { r.Get("/", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetGlobalSettings)) r.Patch("/{id}", dfHandler.AuthHandler(ResourceSettings, PermissionWrite, dfHandler.UpdateGlobalSettings)) diff --git a/deepfence_utils/postgresql/postgresql-db/db.go b/deepfence_utils/postgresql/postgresql-db/db.go index 3e56a3615c..16bc59cfee 100644 --- a/deepfence_utils/postgresql/postgresql-db/db.go +++ b/deepfence_utils/postgresql/postgresql-db/db.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.20.0 +// sqlc v1.21.0 package postgresql_db diff --git a/deepfence_utils/postgresql/postgresql-db/models.go b/deepfence_utils/postgresql/postgresql-db/models.go index 6340876e58..8da0a45762 100644 --- a/deepfence_utils/postgresql/postgresql-db/models.go +++ b/deepfence_utils/postgresql/postgresql-db/models.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.20.0 +// sqlc v1.21.0 package postgresql_db diff --git a/deepfence_utils/postgresql/postgresql-db/queries.sql.go b/deepfence_utils/postgresql/postgresql-db/queries.sql.go index 4c18be5846..3981f09d12 100644 --- a/deepfence_utils/postgresql/postgresql-db/queries.sql.go +++ b/deepfence_utils/postgresql/postgresql-db/queries.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.20.0 +// sqlc v1.21.0 // source: queries.sql package postgresql_db @@ -1030,9 +1030,15 @@ SELECT l.event, l.user_role as role, l.created_at FROM audit_log l -ORDER BY l.created_at DESC +ORDER BY id DESC +OFFSET $1 LIMIT $2 ` +type GetAuditLogsParams struct { + Offset int32 `json:"offset"` + Limit int32 `json:"limit"` +} + type GetAuditLogsRow struct { Event string `json:"event"` Action string `json:"action"` @@ -1043,8 +1049,8 @@ type GetAuditLogsRow struct { CreatedAt time.Time `json:"created_at"` } -func (q *Queries) GetAuditLogs(ctx context.Context) ([]GetAuditLogsRow, error) { - rows, err := q.db.QueryContext(ctx, getAuditLogs) +func (q *Queries) GetAuditLogs(ctx context.Context, arg GetAuditLogsParams) ([]GetAuditLogsRow, error) { + rows, err := q.db.QueryContext(ctx, getAuditLogs, arg.Offset, arg.Limit) if err != nil { return nil, err } diff --git a/deepfence_utils/postgresql/queries.sql b/deepfence_utils/postgresql/queries.sql index 9e08192755..89adf9854f 100644 --- a/deepfence_utils/postgresql/queries.sql +++ b/deepfence_utils/postgresql/queries.sql @@ -533,7 +533,8 @@ SELECT l.event, l.user_role as role, l.created_at FROM audit_log l -ORDER BY l.created_at DESC; +ORDER BY id DESC +OFFSET $1 LIMIT $2; -- name: GetAuditLogsLast5Minutes :many SELECT l.event, From 5febbb8c59a4545a36e66d0641a89143c5a07dbb Mon Sep 17 00:00:00 2001 From: Ramanan Ravikumar Date: Tue, 26 Sep 2023 18:00:59 +0530 Subject: [PATCH 2/2] Pagination for user audit logs api deepfence/enterprise-roadmap#1958 --- deepfence_server/apiDocs/operation.go | 7 ++++-- deepfence_server/handler/audit_log.go | 22 +++++++++++++++++++ deepfence_server/router/router.go | 5 ++++- .../postgresql/postgresql-db/queries.sql.go | 12 ++++++++++ deepfence_utils/postgresql/queries.sql | 4 ++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index 5a9e5abbaf..ed963e6e1b 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -735,9 +735,12 @@ func (d *OpenApiDocs) AddSettingsOperations() { d.AddOperation("updateSetting", http.MethodPatch, "/deepfence/settings/global-settings/{id}", "Update setting", "Update setting", http.StatusNoContent, []string{tagSettings}, bearerToken, new(SettingUpdateRequest), nil) - d.AddOperation("getUserActivityLogs", http.MethodPost, "/deepfence/settings/user-activity-log", - "Get activity logs", "Get activity logs for all users", + d.AddOperation("getUserAuditLogs", http.MethodPost, "/deepfence/settings/user-audit-log", + "Get user audit logs", "Get audit logs for all users", http.StatusOK, []string{tagSettings}, bearerToken, new(GetAuditLogsRequest), new([]postgresqldb.GetAuditLogsRow)) + d.AddOperation("getUserAuditLogsCount", http.MethodGet, "/deepfence/settings/user-audit-log/count", + "Get user audit logs count", "Get user audit logs count", + http.StatusOK, []string{tagSettings}, bearerToken, nil, new(SearchCountResp)) // Scheduled tasks d.AddOperation("getScheduledTasks", http.MethodGet, "/deepfence/scheduled-task", diff --git a/deepfence_server/handler/audit_log.go b/deepfence_server/handler/audit_log.go index 75a1081534..0ebb3d80f9 100644 --- a/deepfence_server/handler/audit_log.go +++ b/deepfence_server/handler/audit_log.go @@ -7,6 +7,7 @@ import ( "time" "github.com/deepfence/ThreatMapper/deepfence_server/model" + reporters_search "github.com/deepfence/ThreatMapper/deepfence_server/reporters/search" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" postgresql_db "github.com/deepfence/ThreatMapper/deepfence_utils/postgresql/postgresql-db" @@ -155,6 +156,27 @@ func (h *Handler) AddAuditLog(namespace string, params postgresql_db.CreateAudit return nil } +func (h *Handler) GetAuditLogsCount(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + pgClient, err := directory.PostgresClient(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to get db connection") + h.respondError(err, w) + return + } + + count, err := pgClient.CountAuditLogs(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to run CountAuditLogs query") + h.respondError(err, w) + return + } + + httpext.JSON(w, http.StatusOK, reporters_search.SearchCountResp{ + Count: int(count), + }) +} + func (h *Handler) GetAuditLogs(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() var req model.GetAuditLogsRequest diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index 03b3980dab..c6248b7124 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -206,7 +206,10 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c }) r.Route("/settings", func(r chi.Router) { - r.Post("/user-activity-log", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetAuditLogs)) + r.Route("/user-audit-log", func(r chi.Router) { + r.Post("/", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetAuditLogs)) + r.Get("/count", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetAuditLogsCount)) + }) r.Route("/global-settings", func(r chi.Router) { r.Get("/", dfHandler.AuthHandler(ResourceSettings, PermissionRead, dfHandler.GetGlobalSettings)) r.Patch("/{id}", dfHandler.AuthHandler(ResourceSettings, PermissionWrite, dfHandler.UpdateGlobalSettings)) diff --git a/deepfence_utils/postgresql/postgresql-db/queries.sql.go b/deepfence_utils/postgresql/postgresql-db/queries.sql.go index 3981f09d12..d9d6fea8c7 100644 --- a/deepfence_utils/postgresql/postgresql-db/queries.sql.go +++ b/deepfence_utils/postgresql/postgresql-db/queries.sql.go @@ -42,6 +42,18 @@ func (q *Queries) CountActiveUsers(ctx context.Context) (int64, error) { return count, err } +const countAuditLogs = `-- name: CountAuditLogs :one +SELECT count(*) +FROM audit_log +` + +func (q *Queries) CountAuditLogs(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countAuditLogs) + var count int64 + err := row.Scan(&count) + return count, err +} + const countCompanies = `-- name: CountCompanies :one SELECT count(*) FROM company diff --git a/deepfence_utils/postgresql/queries.sql b/deepfence_utils/postgresql/queries.sql index 89adf9854f..15e2637486 100644 --- a/deepfence_utils/postgresql/queries.sql +++ b/deepfence_utils/postgresql/queries.sql @@ -536,6 +536,10 @@ FROM audit_log l ORDER BY id DESC OFFSET $1 LIMIT $2; +-- name: CountAuditLogs :one +SELECT count(*) +FROM audit_log; + -- name: GetAuditLogsLast5Minutes :many SELECT l.event, l.action,