From 41442c9302bd00f9dc11cda038ea2d8eb6907386 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Mon, 20 Mar 2023 17:59:01 +0100 Subject: [PATCH 01/12] Do not store inactive repos without any resources --- pipeline/rpc/proto/woodpecker_grpc.pb.go | 83 +++++++++---- server/api/badge.go | 4 +- server/api/repo.go | 74 ++++++++--- server/api/user.go | 58 ++++----- server/forge/bitbucket/bitbucket.go | 48 ++------ server/forge/bitbucket/convert.go | 18 ++- server/forge/bitbucket/convert_test.go | 7 +- server/forge/bitbucket/parse.go | 4 +- .../forge/bitbucketserver/bitbucketserver.go | 24 ++-- server/forge/bitbucketserver/convert.go | 3 +- server/forge/bitbucketserver/convert_test.go | 3 +- server/forge/bitbucketserver/parse.go | 2 +- server/forge/forge.go | 4 - server/forge/gitea/gitea.go | 14 --- server/forge/gitea/gitea_test.go | 14 --- server/forge/gitea/helper.go | 1 + server/forge/github/github.go | 10 -- server/forge/github/github_test.go | 14 --- server/forge/gitlab/convert.go | 6 +- server/forge/gitlab/gitlab.go | 24 ---- server/forge/gogs/gogs.go | 10 -- server/forge/gogs/gogs_test.go | 14 --- server/forge/gogs/helper.go | 1 + server/forge/mocks/forge.go | 28 +---- server/forge/userSyncer.go | 115 ------------------ server/model/repo.go | 1 - server/model/user.go | 3 - server/router/api.go | 2 +- server/router/middleware/session/repo.go | 14 +-- .../migration/015_remove_inactive_repos.go | 32 +++++ server/store/datastore/migration/migration.go | 1 + server/store/datastore/repo.go | 11 +- server/store/mocks/store.go | 2 +- server/web/config.go | 8 -- web/components.d.ts | 3 +- web/src/compositions/useConfig.ts | 2 - web/src/lib/api/index.ts | 1 - web/src/views/RepoAdd.vue | 12 -- 38 files changed, 248 insertions(+), 427 deletions(-) delete mode 100644 server/forge/userSyncer.go create mode 100644 server/store/datastore/migration/015_remove_inactive_repos.go diff --git a/pipeline/rpc/proto/woodpecker_grpc.pb.go b/pipeline/rpc/proto/woodpecker_grpc.pb.go index c79eb5c523..3945ed2f4b 100644 --- a/pipeline/rpc/proto/woodpecker_grpc.pb.go +++ b/pipeline/rpc/proto/woodpecker_grpc.pb.go @@ -1,6 +1,21 @@ +// Copyright 2021 Woodpecker Authors +// Copyright 2011 Drone.IO Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v3.21.12 // source: woodpecker.proto @@ -18,6 +33,20 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + Woodpecker_Version_FullMethodName = "/proto.Woodpecker/Version" + Woodpecker_Next_FullMethodName = "/proto.Woodpecker/Next" + Woodpecker_Init_FullMethodName = "/proto.Woodpecker/Init" + Woodpecker_Wait_FullMethodName = "/proto.Woodpecker/Wait" + Woodpecker_Done_FullMethodName = "/proto.Woodpecker/Done" + Woodpecker_Extend_FullMethodName = "/proto.Woodpecker/Extend" + Woodpecker_Update_FullMethodName = "/proto.Woodpecker/Update" + Woodpecker_Upload_FullMethodName = "/proto.Woodpecker/Upload" + Woodpecker_Log_FullMethodName = "/proto.Woodpecker/Log" + Woodpecker_RegisterAgent_FullMethodName = "/proto.Woodpecker/RegisterAgent" + Woodpecker_ReportHealth_FullMethodName = "/proto.Woodpecker/ReportHealth" +) + // WoodpeckerClient is the client API for Woodpecker service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -45,7 +74,7 @@ func NewWoodpeckerClient(cc grpc.ClientConnInterface) WoodpeckerClient { func (c *woodpeckerClient) Version(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*VersionResponse, error) { out := new(VersionResponse) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Version", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Version_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -54,7 +83,7 @@ func (c *woodpeckerClient) Version(ctx context.Context, in *Empty, opts ...grpc. func (c *woodpeckerClient) Next(ctx context.Context, in *NextRequest, opts ...grpc.CallOption) (*NextResponse, error) { out := new(NextResponse) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Next", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Next_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -63,7 +92,7 @@ func (c *woodpeckerClient) Next(ctx context.Context, in *NextRequest, opts ...gr func (c *woodpeckerClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Init", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Init_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -72,7 +101,7 @@ func (c *woodpeckerClient) Init(ctx context.Context, in *InitRequest, opts ...gr func (c *woodpeckerClient) Wait(ctx context.Context, in *WaitRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Wait", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Wait_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -81,7 +110,7 @@ func (c *woodpeckerClient) Wait(ctx context.Context, in *WaitRequest, opts ...gr func (c *woodpeckerClient) Done(ctx context.Context, in *DoneRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Done", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Done_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -90,7 +119,7 @@ func (c *woodpeckerClient) Done(ctx context.Context, in *DoneRequest, opts ...gr func (c *woodpeckerClient) Extend(ctx context.Context, in *ExtendRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Extend", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Extend_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -99,7 +128,7 @@ func (c *woodpeckerClient) Extend(ctx context.Context, in *ExtendRequest, opts . func (c *woodpeckerClient) Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Update", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Update_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -108,7 +137,7 @@ func (c *woodpeckerClient) Update(ctx context.Context, in *UpdateRequest, opts . func (c *woodpeckerClient) Upload(ctx context.Context, in *UploadRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Upload", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Upload_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -117,7 +146,7 @@ func (c *woodpeckerClient) Upload(ctx context.Context, in *UploadRequest, opts . func (c *woodpeckerClient) Log(ctx context.Context, in *LogRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/Log", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_Log_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -126,7 +155,7 @@ func (c *woodpeckerClient) Log(ctx context.Context, in *LogRequest, opts ...grpc func (c *woodpeckerClient) RegisterAgent(ctx context.Context, in *RegisterAgentRequest, opts ...grpc.CallOption) (*RegisterAgentResponse, error) { out := new(RegisterAgentResponse) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/RegisterAgent", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_RegisterAgent_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -135,7 +164,7 @@ func (c *woodpeckerClient) RegisterAgent(ctx context.Context, in *RegisterAgentR func (c *woodpeckerClient) ReportHealth(ctx context.Context, in *ReportHealthRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/proto.Woodpecker/ReportHealth", in, out, opts...) + err := c.cc.Invoke(ctx, Woodpecker_ReportHealth_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -220,7 +249,7 @@ func _Woodpecker_Version_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Version", + FullMethod: Woodpecker_Version_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Version(ctx, req.(*Empty)) @@ -238,7 +267,7 @@ func _Woodpecker_Next_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Next", + FullMethod: Woodpecker_Next_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Next(ctx, req.(*NextRequest)) @@ -256,7 +285,7 @@ func _Woodpecker_Init_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Init", + FullMethod: Woodpecker_Init_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Init(ctx, req.(*InitRequest)) @@ -274,7 +303,7 @@ func _Woodpecker_Wait_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Wait", + FullMethod: Woodpecker_Wait_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Wait(ctx, req.(*WaitRequest)) @@ -292,7 +321,7 @@ func _Woodpecker_Done_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Done", + FullMethod: Woodpecker_Done_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Done(ctx, req.(*DoneRequest)) @@ -310,7 +339,7 @@ func _Woodpecker_Extend_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Extend", + FullMethod: Woodpecker_Extend_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Extend(ctx, req.(*ExtendRequest)) @@ -328,7 +357,7 @@ func _Woodpecker_Update_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Update", + FullMethod: Woodpecker_Update_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Update(ctx, req.(*UpdateRequest)) @@ -346,7 +375,7 @@ func _Woodpecker_Upload_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Upload", + FullMethod: Woodpecker_Upload_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Upload(ctx, req.(*UploadRequest)) @@ -364,7 +393,7 @@ func _Woodpecker_Log_Handler(srv interface{}, ctx context.Context, dec func(inte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/Log", + FullMethod: Woodpecker_Log_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).Log(ctx, req.(*LogRequest)) @@ -382,7 +411,7 @@ func _Woodpecker_RegisterAgent_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/RegisterAgent", + FullMethod: Woodpecker_RegisterAgent_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).RegisterAgent(ctx, req.(*RegisterAgentRequest)) @@ -400,7 +429,7 @@ func _Woodpecker_ReportHealth_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Woodpecker/ReportHealth", + FullMethod: Woodpecker_ReportHealth_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerServer).ReportHealth(ctx, req.(*ReportHealthRequest)) @@ -464,6 +493,10 @@ var Woodpecker_ServiceDesc = grpc.ServiceDesc{ Metadata: "woodpecker.proto", } +const ( + WoodpeckerAuth_Auth_FullMethodName = "/proto.WoodpeckerAuth/Auth" +) + // WoodpeckerAuthClient is the client API for WoodpeckerAuth service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -481,7 +514,7 @@ func NewWoodpeckerAuthClient(cc grpc.ClientConnInterface) WoodpeckerAuthClient { func (c *woodpeckerAuthClient) Auth(ctx context.Context, in *AuthRequest, opts ...grpc.CallOption) (*AuthResponse, error) { out := new(AuthResponse) - err := c.cc.Invoke(ctx, "/proto.WoodpeckerAuth/Auth", in, out, opts...) + err := c.cc.Invoke(ctx, WoodpeckerAuth_Auth_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -526,7 +559,7 @@ func _WoodpeckerAuth_Auth_Handler(srv interface{}, ctx context.Context, dec func } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.WoodpeckerAuth/Auth", + FullMethod: WoodpeckerAuth_Auth_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(WoodpeckerAuthServer).Auth(ctx, req.(*AuthRequest)) diff --git a/server/api/badge.go b/server/api/badge.go index 8d8bfc0a94..08cb91f864 100644 --- a/server/api/badge.go +++ b/server/api/badge.go @@ -36,8 +36,8 @@ import ( func GetBadge(c *gin.Context) { _store := store.FromContext(c) repo, err := _store.GetRepoName(c.Param("owner") + "/" + c.Param("name")) - if err != nil || !repo.IsActive { - if err == nil || errors.Is(err, types.RecordNotExist) { + if err != nil { + if errors.Is(err, types.RecordNotExist) { c.AbortWithStatus(http.StatusNotFound) return } diff --git a/server/api/repo.go b/server/api/repo.go index e04b0ed7cc..a0a550b855 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -17,13 +17,16 @@ package api import ( "encoding/base32" + "errors" "fmt" "net/http" "strconv" + "time" "github.com/gin-gonic/gin" "github.com/gorilla/securecookie" "github.com/rs/zerolog/log" + "github.com/woodpecker-ci/woodpecker/server/store/types" "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/model" @@ -36,17 +39,37 @@ func PostRepo(c *gin.Context) { forge := server.Config.Services.Forge _store := store.FromContext(c) user := session.User(c) - repo := session.Repo(c) - if repo.IsActive { + owner := c.Param("owner") + name := c.Param("name") + repo, err := _store.GetRepoName(owner + "/" + name) + enabledOnce := err == nil // if there's no error, the repo was found and enabled once already + if err == nil && repo.IsActive { c.String(http.StatusConflict, "Repository is already active.") return + } else if err != nil && !errors.Is(err, types.RecordNotExist) { + c.String(http.StatusInternalServerError, err.Error()) + return } + from, err := forge.Repo(c, user, "0", owner, name) + if err != nil { + c.String(http.StatusInternalServerError, "Could not fetch repository from forge.") + return + } + if !from.Perm.Admin { + c.String(http.StatusForbidden, "User not authorized") + } + + if enabledOnce { + repo.Update(from) + } else { + repo = from + repo.AllowPull = true + repo.CancelPreviousPipelineEvents = server.Config.Pipeline.DefaultCancelPreviousPipelineEvents + } repo.IsActive = true repo.UserID = user.ID - repo.AllowPull = true - repo.CancelPreviousPipelineEvents = server.Config.Pipeline.DefaultCancelPreviousPipelineEvents if repo.Visibility == "" { repo.Visibility = model.VisibilityPublic @@ -81,26 +104,27 @@ func PostRepo(c *gin.Context) { sig, ) - from, err := forge.Repo(c, user, repo.ForgeRemoteID, repo.Owner, repo.Name) - if err == nil { - if repo.FullName != from.FullName { - // create a redirection - err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) - if err != nil { - _ = c.AbortWithError(http.StatusInternalServerError, err) - return - } - } - repo.Update(from) - } - err = forge.Activate(c, user, repo, link) if err != nil { c.String(http.StatusInternalServerError, err.Error()) return } - err = _store.UpdateRepo(repo) + if enabledOnce { + err = _store.UpdateRepo(repo) + } else { + err = _store.CreateRepo(repo) + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + repo.Perm = from.Perm + repo.Perm.Synced = time.Now().Unix() + repo.Perm.UserID = user.ID + repo.Perm.RepoID = repo.ID + repo.Perm.Repo = repo + err = _store.PermUpsert(repo.Perm) if err != nil { c.String(http.StatusInternalServerError, err.Error()) return @@ -244,7 +268,7 @@ func DeleteRepo(c *gin.Context) { _ = c.AbortWithError(http.StatusInternalServerError, err) return } - c.JSON(200, repo) + c.JSON(http.StatusOK, repo) } func RepairRepo(c *gin.Context) { @@ -290,6 +314,11 @@ func RepairRepo(c *gin.Context) { _ = c.AbortWithError(http.StatusInternalServerError, err) return } + repo.Perm = from.Perm + if err := _store.PermUpsert(repo.Perm); err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } if err := forge.Deactivate(c, user, repo, host); err != nil { log.Trace().Err(err).Msgf("deactivate repo '%s' to repair failed", repo.FullName) @@ -338,12 +367,17 @@ func MoveRepo(c *gin.Context) { } repo.Update(from) - errStore := _store.UpdateRepo(repo) if errStore != nil { _ = c.AbortWithError(http.StatusInternalServerError, errStore) return } + repo.Perm = from.Perm + errStore = _store.PermUpsert(repo.Perm) + if errStore != nil { + _ = c.AbortWithError(http.StatusInternalServerError, errStore) + return + } // creates the jwt token used to verify the repository t := token.New(token.HookToken, repo.FullName) diff --git a/server/api/user.go b/server/api/user.go index 6a7cede6a7..0070fc3cc6 100644 --- a/server/api/user.go +++ b/server/api/user.go @@ -18,14 +18,10 @@ import ( "encoding/base32" "net/http" "strconv" - "time" "github.com/gin-gonic/gin" "github.com/gorilla/securecookie" - "github.com/rs/zerolog/log" - "github.com/woodpecker-ci/woodpecker/server" - "github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server/store" @@ -38,35 +34,10 @@ func GetSelf(c *gin.Context) { func GetFeed(c *gin.Context) { _store := store.FromContext(c) - _forge := server.Config.Services.Forge user := session.User(c) latest, _ := strconv.ParseBool(c.Query("latest")) - if time.Unix(user.Synced, 0).Add(time.Hour * 72).Before(time.Now()) { - log.Debug().Msgf("sync begin: %s", user.Login) - - user.Synced = time.Now().Unix() - if err := _store.UpdateUser(user); err != nil { - log.Error().Err(err).Msg("UpdateUser") - return - } - - config := ToConfig(c) - - sync := forge.Syncer{ - Forge: _forge, - Store: _store, - Perms: _store, - Match: forge.NamespaceFilter(config.OwnersWhitelist), - } - if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil { - log.Debug().Msgf("sync error: %s: %s", user.Login, err) - } else { - log.Debug().Msgf("sync complete: %s", user.Login) - } - } - if latest { feed, err := _store.RepoListLatest(user) if err != nil { @@ -91,9 +62,8 @@ func GetRepos(c *gin.Context) { user := session.User(c) all, _ := strconv.ParseBool(c.Query("all")) - flush, _ := strconv.ParseBool(c.Query("flush")) - if flush || time.Unix(user.Synced, 0).Add(time.Hour*72).Before(time.Now()) { + /*if flush || time.Unix(user.Synced, 0).Add(time.Hour*72).Before(time.Now()) { log.Debug().Msgf("sync begin: %s", user.Login) user.Synced = time.Now().Unix() if err := _store.UpdateUser(user); err != nil { @@ -115,21 +85,41 @@ func GetRepos(c *gin.Context) { } else { log.Debug().Msgf("sync complete: %s", user.Login) } - } + }*/ - repos, err := _store.RepoList(user, true) + dbRepos, err := _store.RepoList(user, true) if err != nil { c.String(http.StatusInternalServerError, "Error fetching repository list. %s", err) return } if all { + active := map[string]bool{} + for _, r := range dbRepos { + active[r.FullName] = r.IsActive + } + + _repos, err := _forge.Repos(c, user) + if err != nil { + c.String(http.StatusInternalServerError, "Error fetching repository list. %s", err) + return + } + var repos []*model.Repo + for _, r := range _repos { + if r.Perm.Push { + if active[r.FullName] { + r.IsActive = true + } + repos = append(repos, r) + } + } + c.JSON(http.StatusOK, repos) return } active := make([]*model.Repo, 0) - for _, repo := range repos { + for _, repo := range dbRepos { if repo.IsActive { active = append(active, repo) } diff --git a/server/forge/bitbucket/bitbucket.go b/server/forge/bitbucket/bitbucket.go index ba536b5f5b..81c8401bcf 100644 --- a/server/forge/bitbucket/bitbucket.go +++ b/server/forge/bitbucket/bitbucket.go @@ -148,11 +148,16 @@ func (c *config) Repo(ctx context.Context, u *model.User, remoteID model.ForgeRe if remoteID.IsValid() { name = string(remoteID) } - repo, err := c.newClient(ctx, u).FindRepo(owner, name) + client := c.newClient(ctx, u) + repo, err := client.FindRepo(owner, name) + if err != nil { + return nil, err + } + perm, err := client.GetPermission(repo.FullName) if err != nil { return nil, err } - return convertRepo(repo), nil + return convertRepo(repo, perm), nil } // Repos returns a list of all repositories for Bitbucket account, including @@ -176,44 +181,17 @@ func (c *config) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error return all, err } for _, repo := range repos { - all = append(all, convertRepo(repo)) + perm, err := client.GetPermission(repo.FullName) + if err != nil { + return nil, err + } + + all = append(all, convertRepo(repo, perm)) } } return all, nil } -// Perm returns the user permissions for the named repository. Because Bitbucket -// does not have an endpoint to access user permissions, we attempt to fetch -// the repository hook list, which is restricted to administrators to calculate -// administrative access to a repository. -func (c *config) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) { - client := c.newClient(ctx, u) - - perms := new(model.Perm) - repo, err := client.FindRepo(r.Owner, r.Name) - if err != nil { - return perms, err - } - - perm, err := client.GetPermission(repo.FullName) - if err != nil { - return perms, err - } - - switch perm.Permission { - case "admin": - perms.Admin = true - fallthrough - case "write": - perms.Push = true - fallthrough - default: - perms.Pull = true - } - - return perms, nil -} - // File fetches the file from the Bitbucket repository and returns its contents. func (c *config) File(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]byte, error) { config, err := c.newClient(ctx, u).FindSource(r.Owner, r.Name, p.Commit, f) diff --git a/server/forge/bitbucket/convert.go b/server/forge/bitbucket/convert.go index 052f20b2a5..8449e75a37 100644 --- a/server/forge/bitbucket/convert.go +++ b/server/forge/bitbucket/convert.go @@ -48,7 +48,7 @@ func convertStatus(status model.StatusValue) string { // convertRepo is a helper function used to convert a Bitbucket repository // structure to the common Woodpecker repository structure. -func convertRepo(from *internal.Repo) *model.Repo { +func convertRepo(from *internal.Repo, perm *internal.RepoPerm) *model.Repo { repo := model.Repo{ ForgeRemoteID: model.ForgeRemoteID(from.UUID), Clone: cloneLink(from), @@ -60,6 +60,7 @@ func convertRepo(from *internal.Repo) *model.Repo { Avatar: from.Owner.Links.Avatar.Href, SCMKind: model.SCMKind(from.Scm), Branch: "master", + Perm: convertPerm(perm), } if repo.SCMKind == model.RepoHg { repo.Branch = "default" @@ -67,6 +68,21 @@ func convertRepo(from *internal.Repo) *model.Repo { return &repo } +func convertPerm(from *internal.RepoPerm) *model.Perm { + perms := new(model.Perm) + switch from.Permission { + case "admin": + perms.Admin = true + fallthrough + case "write": + perms.Push = true + fallthrough + default: + perms.Pull = true + } + return perms +} + // cloneLink is a helper function that tries to extract the clone url from the // repository object. func cloneLink(repo *internal.Repo) string { diff --git a/server/forge/bitbucket/convert_test.go b/server/forge/bitbucket/convert_test.go index 5ac86c91ce..65efded7d9 100644 --- a/server/forge/bitbucket/convert_test.go +++ b/server/forge/bitbucket/convert_test.go @@ -52,8 +52,11 @@ func Test_helper(t *testing.T) { } from.Owner.Links.Avatar.Href = "http://..." from.Links.HTML.Href = "https://bitbucket.org/foo/bar" + fromPerm := &internal.RepoPerm{ + Permission: "write", + } - to := convertRepo(from) + to := convertRepo(from, fromPerm) g.Assert(to.Avatar).Equal(from.Owner.Links.Avatar.Href) g.Assert(to.FullName).Equal(from.FullName) g.Assert(to.Owner).Equal("octocat") @@ -63,6 +66,8 @@ func Test_helper(t *testing.T) { g.Assert(to.IsSCMPrivate).Equal(from.IsPrivate) g.Assert(to.Clone).Equal(from.Links.HTML.Href) g.Assert(to.Link).Equal(from.Links.HTML.Href) + g.Assert(to.Perm.Push).IsTrue() + g.Assert(to.Perm.Admin).IsFalse() }) g.It("should convert team", func() { diff --git a/server/forge/bitbucket/parse.go b/server/forge/bitbucket/parse.go index 5e277a1048..def263678a 100644 --- a/server/forge/bitbucket/parse.go +++ b/server/forge/bitbucket/parse.go @@ -59,7 +59,7 @@ func parsePushHook(payload []byte) (*model.Repo, *model.Pipeline, error) { if change.New.Target.Hash == "" { continue } - return convertRepo(&hook.Repo), convertPushHook(&hook, &change), nil + return convertRepo(&hook.Repo, &internal.RepoPerm{}), convertPushHook(&hook, &change), nil } return nil, nil, nil } @@ -75,5 +75,5 @@ func parsePullHook(payload []byte) (*model.Repo, *model.Pipeline, error) { if hook.PullRequest.State != stateOpen { return nil, nil, nil } - return convertRepo(&hook.Repo), convertPullHook(&hook), nil + return convertRepo(&hook.Repo, &internal.RepoPerm{}), convertPullHook(&hook), nil } diff --git a/server/forge/bitbucketserver/bitbucketserver.go b/server/forge/bitbucketserver/bitbucketserver.go index c6d3528a12..a67ec6a141 100644 --- a/server/forge/bitbucketserver/bitbucketserver.go +++ b/server/forge/bitbucketserver/bitbucketserver.go @@ -154,32 +154,36 @@ func (*Config) TeamPerm(_ *model.User, _ string) (*model.Perm, error) { } func (c *Config) Repo(ctx context.Context, u *model.User, _ model.ForgeRemoteID, owner, name string) (*model.Repo, error) { - repo, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).FindRepo(owner, name) + client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token) + repo, err := client.FindRepo(owner, name) + if err != nil { + return nil, err + } + perm, err := client.FindRepoPerms(repo.Project.Key, repo.Name) if err != nil { return nil, err } - return convertRepo(repo), nil + return convertRepo(repo, perm), nil } func (c *Config) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) { - repos, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).FindRepos() + client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token) + repos, err := client.FindRepos() if err != nil { return nil, err } var all []*model.Repo for _, repo := range repos { - all = append(all, convertRepo(repo)) + perm, err := client.FindRepoPerms(repo.Project.Key, repo.Name) + if err != nil { + return nil, err + } + all = append(all, convertRepo(repo, perm)) } return all, nil } -func (c *Config) Perm(ctx context.Context, u *model.User, repo *model.Repo) (*model.Perm, error) { - client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token) - - return client.FindRepoPerms(repo.Owner, repo.Name) -} - func (c *Config) File(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]byte, error) { client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token) diff --git a/server/forge/bitbucketserver/convert.go b/server/forge/bitbucketserver/convert.go index d6dab05189..43b0ea4a1b 100644 --- a/server/forge/bitbucketserver/convert.go +++ b/server/forge/bitbucketserver/convert.go @@ -50,7 +50,7 @@ func convertStatus(status model.StatusValue) string { // convertRepo is a helper function used to convert a Bitbucket server repository // structure to the common Woodpecker repository structure. -func convertRepo(from *internal.Repo) *model.Repo { +func convertRepo(from *internal.Repo, perm *model.Perm) *model.Repo { repo := model.Repo{ ForgeRemoteID: model.ForgeRemoteID(fmt.Sprint(from.ID)), Name: from.Slug, @@ -59,6 +59,7 @@ func convertRepo(from *internal.Repo) *model.Repo { SCMKind: model.RepoGit, IsSCMPrivate: true, // Since we have to use Netrc it has to always be private :/ FullName: fmt.Sprintf("%s/%s", from.Project.Key, from.Slug), + Perm: perm, } for _, item := range from.Links.Clone { diff --git a/server/forge/bitbucketserver/convert_test.go b/server/forge/bitbucketserver/convert_test.go index db97478930..adfe635ba6 100644 --- a/server/forge/bitbucketserver/convert_test.go +++ b/server/forge/bitbucketserver/convert_test.go @@ -47,7 +47,7 @@ func Test_helper(t *testing.T) { from.Links.Self = append(from.Links.Self, selfRef) - to := convertRepo(from) + to := convertRepo(from, &model.Perm{Pull: true}) g.Assert(to.FullName).Equal("octocat/hello-world") g.Assert(to.Owner).Equal("octocat") g.Assert(to.Name).Equal("hello-world") @@ -56,6 +56,7 @@ func Test_helper(t *testing.T) { g.Assert(to.IsSCMPrivate).Equal(true) g.Assert(to.Clone).Equal("https://server.org/foo/bar.git") g.Assert(to.Link).Equal("https://server.org/foo/bar") + g.Assert(to.Perm.Pull).IsTrue() }) g.It("should convert user", func() { diff --git a/server/forge/bitbucketserver/parse.go b/server/forge/bitbucketserver/parse.go index e66015b82f..be10ebffc2 100644 --- a/server/forge/bitbucketserver/parse.go +++ b/server/forge/bitbucketserver/parse.go @@ -31,7 +31,7 @@ func parseHook(r *http.Request, baseURL string) (*model.Repo, *model.Pipeline, e return nil, nil, err } pipeline := convertPushHook(hook, baseURL) - repo := convertRepo(&hook.Repository) + repo := convertRepo(&hook.Repository, &model.Perm{}) return repo, pipeline, nil } diff --git a/server/forge/forge.go b/server/forge/forge.go index ca2c679b0a..971fdc6f34 100644 --- a/server/forge/forge.go +++ b/server/forge/forge.go @@ -50,10 +50,6 @@ type Forge interface { // Repos fetches a list of repos from the forge. Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) - // Perm fetches the named repository permissions from - // the forge for the specified user. - Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) - // File fetches a file from the forge repository and returns in string // format. File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) diff --git a/server/forge/gitea/gitea.go b/server/forge/gitea/gitea.go index 9faa5452d0..080d3865f4 100644 --- a/server/forge/gitea/gitea.go +++ b/server/forge/gitea/gitea.go @@ -268,20 +268,6 @@ func (c *Gitea) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) }) } -// Perm returns the user permissions for the named Gitea repository. -func (c *Gitea) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) { - client, err := c.newClientToken(ctx, u.Token) - if err != nil { - return nil, err - } - - repo, _, err := client.GetRepo(r.Owner, r.Name) - if err != nil { - return nil, err - } - return toPerm(repo.Permissions), nil -} - // File fetches the file from the Gitea repository and returns its contents. func (c *Gitea) File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) { client, err := c.newClientToken(ctx, u.Token) diff --git a/server/forge/gitea/gitea_test.go b/server/forge/gitea/gitea_test.go index 205f3da383..eb9b6d57f8 100644 --- a/server/forge/gitea/gitea_test.go +++ b/server/forge/gitea/gitea_test.go @@ -100,20 +100,6 @@ func Test_gitea(t *testing.T) { }) }) - g.Describe("Requesting repository permissions", func() { - g.It("Should return the permission details", func() { - perm, err := c.Perm(ctx, fakeUser, fakeRepo) - g.Assert(err).IsNil() - g.Assert(perm.Admin).IsTrue() - g.Assert(perm.Push).IsTrue() - g.Assert(perm.Pull).IsTrue() - }) - g.It("Should handle a not found error", func() { - _, err := c.Perm(ctx, fakeUser, fakeRepoNotFound) - g.Assert(err).IsNotNil() - }) - }) - g.Describe("Requesting a repository list", func() { g.It("Should return the repository list", func() { repos, err := c.Repos(ctx, fakeUser) diff --git a/server/forge/gitea/helper.go b/server/forge/gitea/helper.go index fb792581b3..5fe697f7dd 100644 --- a/server/forge/gitea/helper.go +++ b/server/forge/gitea/helper.go @@ -47,6 +47,7 @@ func toRepo(from *gitea.Repository) *model.Repo { IsSCMPrivate: from.Private || from.Owner.Visibility != gitea.VisibleTypePublic, Clone: from.CloneURL, Branch: from.DefaultBranch, + Perm: toPerm(from.Permissions), } } diff --git a/server/forge/github/github.go b/server/forge/github/github.go index 8bafeef512..34dea4ca55 100644 --- a/server/forge/github/github.go +++ b/server/forge/github/github.go @@ -209,16 +209,6 @@ func (c *client) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error return repos, nil } -// Perm returns the user permissions for the named GitHub repository. -func (c *client) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) { - client := c.newClientToken(ctx, u.Token) - repo, _, err := client.Repositories.Get(ctx, r.Owner, r.Name) - if err != nil { - return nil, err - } - return convertPerm(repo.GetPermissions()), nil -} - // File fetches the file from the GitHub repository and returns its contents. func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) { client := c.newClientToken(ctx, u.Token) diff --git a/server/forge/github/github_test.go b/server/forge/github/github_test.go index 9126970726..b5a952ad9c 100644 --- a/server/forge/github/github_test.go +++ b/server/forge/github/github_test.go @@ -94,20 +94,6 @@ func Test_github(t *testing.T) { }) }) - g.Describe("Requesting repository permissions", func() { - g.It("Should return the permission details", func() { - perm, err := c.Perm(ctx, fakeUser, fakeRepo) - g.Assert(err).IsNil() - g.Assert(perm.Admin).IsTrue() - g.Assert(perm.Push).IsTrue() - g.Assert(perm.Pull).IsTrue() - }) - g.It("Should handle a not found error", func() { - _, err := c.Perm(ctx, fakeUser, fakeRepoNotFound) - g.Assert(err).IsNotNil() - }) - }) - g.It("Should return a user repository list") g.It("Should return a user team list") diff --git a/server/forge/gitlab/convert.go b/server/forge/gitlab/convert.go index 40f181e057..4298dc6ebf 100644 --- a/server/forge/gitlab/convert.go +++ b/server/forge/gitlab/convert.go @@ -33,7 +33,6 @@ const ( func (g *GitLab) convertGitLabRepo(_repo *gitlab.Project) (*model.Repo, error) { parts := strings.Split(_repo.PathWithNamespace, "/") - // TODO(648) save repo id (support nested repos) owner := strings.Join(parts[:len(parts)-1], "/") name := parts[len(parts)-1] repo := &model.Repo{ @@ -47,6 +46,11 @@ func (g *GitLab) convertGitLabRepo(_repo *gitlab.Project) (*model.Repo, error) { Branch: _repo.DefaultBranch, Visibility: model.RepoVisibly(_repo.Visibility), IsSCMPrivate: !_repo.Public, + Perm: &model.Perm{ + Pull: isRead(_repo), + Push: isWrite(_repo), + Admin: isAdmin(_repo), + }, } if len(repo.Branch) == 0 { // TODO: do we need that? diff --git a/server/forge/gitlab/gitlab.go b/server/forge/gitlab/gitlab.go index b75469ee8f..e973146791 100644 --- a/server/forge/gitlab/gitlab.go +++ b/server/forge/gitlab/gitlab.go @@ -335,30 +335,6 @@ func (g *GitLab) PullRequests(ctx context.Context, u *model.User, r *model.Repo, return result, err } -// Perm fetches the named repository from the forge. -func (g *GitLab) Perm(ctx context.Context, user *model.User, r *model.Repo) (*model.Perm, error) { - client, err := newClient(g.URL, user.Token, g.SkipVerify) - if err != nil { - return nil, err - } - repo, err := g.getProject(ctx, client, r.Owner, r.Name) - if err != nil { - return nil, err - } - - // repo owner is granted full access - if repo.Owner != nil && repo.Owner.Username == user.Login { - return &model.Perm{Push: true, Pull: true, Admin: true}, nil - } - - // return permission for current user - return &model.Perm{ - Pull: isRead(repo), - Push: isWrite(repo), - Admin: isAdmin(repo), - }, nil -} - // File fetches a file from the forge repository and returns in string format. func (g *GitLab) File(ctx context.Context, user *model.User, repo *model.Repo, pipeline *model.Pipeline, fileName string) ([]byte, error) { client, err := newClient(g.URL, user.Token, g.SkipVerify) diff --git a/server/forge/gogs/gogs.go b/server/forge/gogs/gogs.go index 22781165e3..848fde318d 100644 --- a/server/forge/gogs/gogs.go +++ b/server/forge/gogs/gogs.go @@ -176,16 +176,6 @@ func (c *client) Repos(_ context.Context, u *model.User) ([]*model.Repo, error) return repos, err } -// Perm returns the user permissions for the named Gogs repository. -func (c *client) Perm(_ context.Context, u *model.User, r *model.Repo) (*model.Perm, error) { - client := c.newClientToken(u.Token) - repo, err := client.GetRepo(r.Owner, r.Name) - if err != nil { - return nil, err - } - return toPerm(repo.Permissions), nil -} - // File fetches the file from the Gogs repository and returns its contents. func (c *client) File(_ context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) { client := c.newClientToken(u.Token) diff --git a/server/forge/gogs/gogs_test.go b/server/forge/gogs/gogs_test.go index 48c38cfcc6..d09cdaae00 100644 --- a/server/forge/gogs/gogs_test.go +++ b/server/forge/gogs/gogs_test.go @@ -101,20 +101,6 @@ func Test_gogs(t *testing.T) { }) }) - g.Describe("Requesting repository permissions", func() { - g.It("Should return the permission details", func() { - perm, err := c.Perm(ctx, fakeUser, fakeRepo) - g.Assert(err).IsNil() - g.Assert(perm.Admin).IsTrue() - g.Assert(perm.Push).IsTrue() - g.Assert(perm.Pull).IsTrue() - }) - g.It("Should handle a not found error", func() { - _, err := c.Perm(ctx, fakeUser, fakeRepoNotFound) - g.Assert(err).IsNotNil() - }) - }) - g.Describe("Requesting a repository list", func() { g.It("Should return the repository list", func() { repos, err := c.Repos(ctx, fakeUser) diff --git a/server/forge/gogs/helper.go b/server/forge/gogs/helper.go index 0b77daa443..c3890aba5c 100644 --- a/server/forge/gogs/helper.go +++ b/server/forge/gogs/helper.go @@ -46,6 +46,7 @@ func toRepo(from *gogs.Repository, privateMode bool) *model.Repo { IsSCMPrivate: from.Private || privateMode, Clone: from.CloneURL, Branch: from.DefaultBranch, + Perm: toPerm(from.Permissions), } } diff --git a/server/forge/mocks/forge.go b/server/forge/mocks/forge.go index a684d33661..864c56fdf7 100644 --- a/server/forge/mocks/forge.go +++ b/server/forge/mocks/forge.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.23.0. DO NOT EDIT. package mocks @@ -300,32 +300,6 @@ func (_m *Forge) OrgMembership(ctx context.Context, u *model.User, owner string) return r0, r1 } -// Perm provides a mock function with given fields: ctx, u, r -func (_m *Forge) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) { - ret := _m.Called(ctx, u, r) - - var r0 *model.Perm - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo) (*model.Perm, error)); ok { - return rf(ctx, u, r) - } - if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo) *model.Perm); ok { - r0 = rf(ctx, u, r) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Perm) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *model.User, *model.Repo) error); ok { - r1 = rf(ctx, u, r) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // PullRequests provides a mock function with given fields: ctx, u, r, p func (_m *Forge) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) { ret := _m.Called(ctx, u, r, p) diff --git a/server/forge/userSyncer.go b/server/forge/userSyncer.go deleted file mode 100644 index 83b99fce12..0000000000 --- a/server/forge/userSyncer.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// Copyright 2018 Drone.IO Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package forge - -import ( - "context" - "fmt" - "time" - - "github.com/woodpecker-ci/woodpecker/server/model" - "github.com/woodpecker-ci/woodpecker/server/store" -) - -// UserSyncer syncs the user repository and permissions. -type UserSyncer interface { - Sync(ctx context.Context, user *model.User) error -} - -type Syncer struct { - Forge Forge - Store store.Store - Perms model.PermStore - Match FilterFunc -} - -// FilterFunc can be used to filter which repositories are -// synchronized with the local datastore. -type FilterFunc func(*model.Repo) bool - -func NamespaceFilter(namespaces map[string]bool) FilterFunc { - if len(namespaces) == 0 { - return noopFilter - } - return func(repo *model.Repo) bool { - return namespaces[repo.Owner] - } -} - -// noopFilter is a filter function that always returns true. -func noopFilter(*model.Repo) bool { - return true -} - -// SetFilter sets the filter function. -func (s *Syncer) SetFilter(fn FilterFunc) { - s.Match = fn -} - -func (s *Syncer) Sync(ctx context.Context, user *model.User, flatPermissions bool) error { - unix := time.Now().Unix() - (3601) // force immediate expiration. note 1 hour expiration is hard coded at the moment - repos, err := s.Forge.Repos(ctx, user) - if err != nil { - return err - } - - forgeRepos := make([]*model.Repo, 0, len(repos)) - for _, repo := range repos { - if s.Match(repo) { - repo.Perm = &model.Perm{ - UserID: user.ID, - RepoID: repo.ID, - Repo: repo, - Synced: unix, - } - - // TODO(485) temporary workaround to not hit api rate limits - if flatPermissions { - repo.Perm.Pull = true - repo.Perm.Push = true - repo.Perm.Admin = true - } else { - forgePerm, err := s.Forge.Perm(ctx, user, repo) - if err != nil { - return fmt.Errorf("could not fetch permission of repo '%s': %w", repo.FullName, err) - } - repo.Perm.Pull = forgePerm.Pull - repo.Perm.Push = forgePerm.Push - repo.Perm.Admin = forgePerm.Admin - } - - forgeRepos = append(forgeRepos, repo) - } - } - - err = s.Store.RepoBatch(forgeRepos) - if err != nil { - return err - } - - // this is here as a precaution. I want to make sure that if an api - // call to the version control system fails and (for some reason) returns - // an empty list, we don't wipe out the user repository permissions. - // - // the side-effect of this code is that a user with 1 repository whose - // access is removed will still display in the feed, but they will not - // be able to access the actual repository data. - if len(repos) == 0 { - return nil - } - - return s.Perms.PermFlush(user, unix) -} diff --git a/server/model/repo.go b/server/model/repo.go index 7c2cb702c5..175102731d 100644 --- a/server/model/repo.go +++ b/server/model/repo.go @@ -40,7 +40,6 @@ type Repo struct { Visibility RepoVisibly `json:"visibility" xorm:"varchar(10) 'repo_visibility'"` IsSCMPrivate bool `json:"private" xorm:"repo_private"` IsTrusted bool `json:"trusted" xorm:"repo_trusted"` - IsStarred bool `json:"starred,omitempty" xorm:"-"` IsGated bool `json:"gated" xorm:"repo_gated"` IsActive bool `json:"active" xorm:"repo_active"` AllowPull bool `json:"allow_pr" xorm:"repo_allow_pr"` diff --git a/server/model/user.go b/server/model/user.go index 4f10531dac..5cf21c7686 100644 --- a/server/model/user.go +++ b/server/model/user.go @@ -56,9 +56,6 @@ type User struct { // the avatar url for this user. Avatar string `json:"avatar_url" xorm:" varchar(500) 'user_avatar'"` - // Synced is the timestamp when the user was synced with the forge. - Synced int64 `json:"synced" xorm:"user_synced"` - // Admin indicates the user is a system administrator. // // NOTE: If the username is part of the WOODPECKER_ADMINS diff --git a/server/router/api.go b/server/router/api.go index dcb7468bf4..f841d933da 100644 --- a/server/router/api.go +++ b/server/router/api.go @@ -61,6 +61,7 @@ func apiRoutes(e *gin.Engine) { } } + apiBase.POST("/repos/:owner/:name", session.MustUser(), api.PostRepo) repoBase := apiBase.Group("/repos/:owner/:name") { repoBase.Use(session.SetRepo()) @@ -72,7 +73,6 @@ func apiRoutes(e *gin.Engine) { { repo.Use(session.MustPull) - repo.POST("", session.MustRepoAdmin(), api.PostRepo) repo.GET("", api.GetRepo) repo.GET("/branches", api.GetRepoBranches) diff --git a/server/router/middleware/session/repo.go b/server/router/middleware/session/repo.go index 36291d7caf..309cf63042 100644 --- a/server/router/middleware/session/repo.go +++ b/server/router/middleware/session/repo.go @@ -21,8 +21,8 @@ import ( "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" - "github.com/woodpecker-ci/woodpecker/server" + "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store/types" @@ -94,8 +94,7 @@ func SetPerm() gin.HandlerFunc { repo := Repo(c) perm := new(model.Perm) - switch { - case user != nil: + if user != nil { var err error perm, err = _store.PermFind(user, repo) if err != nil { @@ -103,10 +102,12 @@ func SetPerm() gin.HandlerFunc { user.Login, repo.FullName, err) } if time.Unix(perm.Synced, 0).Add(time.Hour).Before(time.Now()) { - perm, err = server.Config.Services.Forge.Perm(c, user, repo) + _repo, err := server.Config.Services.Forge.Repo(c, user, repo.ForgeRemoteID, repo.Owner, repo.Name) if err == nil { log.Debug().Msgf("Synced user permission for %s %s", user.Login, repo.FullName) + perm = _repo.Perm perm.Repo = repo + perm.RepoID = repo.ID perm.UserID = user.ID perm.Synced = time.Now().Unix() if err := _store.PermUpsert(perm); err != nil { @@ -127,10 +128,7 @@ func SetPerm() gin.HandlerFunc { perm.Admin = true } - switch { - case repo.Visibility == model.VisibilityPublic: - perm.Pull = true - case repo.Visibility == model.VisibilityInternal && user != nil: + if repo.Visibility == model.VisibilityPublic || (repo.Visibility == model.VisibilityInternal && user != nil) { perm.Pull = true } diff --git a/server/store/datastore/migration/015_remove_inactive_repos.go b/server/store/datastore/migration/015_remove_inactive_repos.go new file mode 100644 index 0000000000..76491b6912 --- /dev/null +++ b/server/store/datastore/migration/015_remove_inactive_repos.go @@ -0,0 +1,32 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migration + +import ( + "xorm.io/xorm" +) + +var removeInactiveRepos = task{ + name: "remove-inactive-repos", + required: true, + fn: func(sess *xorm.Session) error { + _, err := sess.Table("repos").Where("repo_active = ?", false).Delete() + if err != nil { + return err + } + + return dropTableColumns(sess, "users", "user_synced") + }, +} diff --git a/server/store/datastore/migration/migration.go b/server/store/datastore/migration/migration.go index f166bbc5cf..5c084b902f 100644 --- a/server/store/datastore/migration/migration.go +++ b/server/store/datastore/migration/migration.go @@ -43,6 +43,7 @@ var migrationTasks = []*task{ &renameRemoteToForge, &renameForgeIDToForgeRemoteID, &removeActiveFromUsers, + &removeInactiveRepos, } var allBeans = []interface{}{ diff --git a/server/store/datastore/repo.go b/server/store/datastore/repo.go index ae5b27fa16..df7b4ebf5e 100644 --- a/server/store/datastore/repo.go +++ b/server/store/datastore/repo.go @@ -155,7 +155,6 @@ func (s storage) RepoList(user *model.User, owned bool) ([]*model.Repo, error) { } // RepoBatch Sync batch of repos from SCM (with permissions) to store (create if not exist else update) -// TODO: only store activated repos ... func (s storage) RepoBatch(repos []*model.Repo) error { sess := s.engine.NewSession() defer sess.Close() @@ -179,6 +178,7 @@ func (s storage) RepoBatch(repos []*model.Repo) error { } } + // If it exists, just update. If it does not exist, it is not active, so we don't store it. if exist { if repos[i].FullName != repo.FullName { // create redirection @@ -190,7 +190,7 @@ func (s storage) RepoBatch(repos []*model.Repo) error { if repos[i].ForgeRemoteID.IsValid() { if _, err := sess. Where("forge_remote_id = ?", repos[i].ForgeRemoteID). - Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_id"). + Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_remote_id"). Update(repos[i]); err != nil { return err } @@ -198,7 +198,7 @@ func (s storage) RepoBatch(repos []*model.Repo) error { if _, err := sess. Where("repo_owner = ?", repos[i].Owner). And(" repo_name = ?", repos[i].Name). - Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_id"). + Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_remote_id"). Update(repos[i]); err != nil { return err } @@ -210,11 +210,6 @@ func (s storage) RepoBatch(repos []*model.Repo) error { if err != nil { return err } - } else { - // only Insert on single object ref set auto created ID back to object - if _, err := sess.Insert(repos[i]); err != nil { - return err - } } if repos[i].Perm != nil { diff --git a/server/store/mocks/store.go b/server/store/mocks/store.go index 82d513e2fb..32dba86b9f 100644 --- a/server/store/mocks/store.go +++ b/server/store/mocks/store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.23.0. DO NOT EDIT. package mocks diff --git a/server/web/config.go b/server/web/config.go index e08b563c4c..e2ce1dfc15 100644 --- a/server/web/config.go +++ b/server/web/config.go @@ -18,7 +18,6 @@ import ( "encoding/json" "net/http" "text/template" - "time" "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" @@ -40,15 +39,9 @@ func Config(c *gin.Context) { ).Sign(user.Hash) } - var syncing bool - if user != nil { - syncing = time.Unix(user.Synced, 0).Add(time.Hour * 72).Before(time.Now()) - } - configData := map[string]interface{}{ "user": user, "csrf": csrf, - "syncing": syncing, "docs": server.Config.Server.Docs, "version": version.String(), "forge": server.Config.Services.Forge.Name(), @@ -73,7 +66,6 @@ func Config(c *gin.Context) { const configTemplate = ` window.WOODPECKER_USER = {{ json .user }}; -window.WOODPECKER_SYNC = {{ .syncing }}; window.WOODPECKER_CSRF = "{{ .csrf }}"; window.WOODPECKER_VERSION = "{{ .version }}"; window.WOODPECKER_DOCS = "{{ .docs }}"; diff --git a/web/components.d.ts b/web/components.d.ts index 38640e0f47..3a34eb3f32 100644 --- a/web/components.d.ts +++ b/web/components.d.ts @@ -20,7 +20,6 @@ declare module '@vue/runtime-core' { Button: typeof import('./src/components/atomic/Button.vue')['default'] Checkbox: typeof import('./src/components/form/Checkbox.vue')['default'] CheckboxesField: typeof import('./src/components/form/CheckboxesField.vue')['default'] - copy: typeof import('./src/components/admin/settings/AdminAgentsTab copy.vue')['default'] CronTab: typeof import('./src/components/repo/settings/CronTab.vue')['default'] DeployPipelinePopup: typeof import('./src/components/layout/popups/DeployPipelinePopup.vue')['default'] DocsLink: typeof import('./src/components/atomic/DocsLink.vue')['default'] @@ -62,10 +61,10 @@ declare module '@vue/runtime-core' { IMdiGestureTap: typeof import('~icons/mdi/gesture-tap')['default'] IMdiGithub: typeof import('~icons/mdi/github')['default'] IMdiLoading: typeof import('~icons/mdi/loading')['default'] - IMdiSync: typeof import('~icons/mdi/sync')['default'] IMdiSourceBranch: typeof import('~icons/mdi/source-branch')['default'] IMdisourceCommit: typeof import('~icons/mdi/source-commit')['default'] IMdiSourcePull: typeof import('~icons/mdi/source-pull')['default'] + IMdiSync: typeof import('~icons/mdi/sync')['default'] IMdiTagOutline: typeof import('~icons/mdi/tag-outline')['default'] InputField: typeof import('./src/components/form/InputField.vue')['default'] IPhGitlabLogoSimpleFill: typeof import('~icons/ph/gitlab-logo-simple-fill')['default'] diff --git a/web/src/compositions/useConfig.ts b/web/src/compositions/useConfig.ts index d019cb57ec..3875f3aaf1 100644 --- a/web/src/compositions/useConfig.ts +++ b/web/src/compositions/useConfig.ts @@ -3,7 +3,6 @@ import { User } from '~/lib/api/types'; declare global { interface Window { WOODPECKER_USER: User | undefined; - WOODPECKER_SYNC: boolean | undefined; WOODPECKER_DOCS: string | undefined; WOODPECKER_VERSION: string | undefined; WOODPECKER_CSRF: string | undefined; @@ -13,7 +12,6 @@ declare global { export default () => ({ user: window.WOODPECKER_USER || null, - syncing: window.WOODPECKER_SYNC || null, docs: window.WOODPECKER_DOCS || null, version: window.WOODPECKER_VERSION, csrf: window.WOODPECKER_CSRF || null, diff --git a/web/src/lib/api/index.ts b/web/src/lib/api/index.ts index 600ad84907..ad26eeb659 100644 --- a/web/src/lib/api/index.ts +++ b/web/src/lib/api/index.ts @@ -20,7 +20,6 @@ import { type RepoListOptions = { all?: boolean; - flush?: boolean; }; type PipelineOptions = { diff --git a/web/src/views/RepoAdd.vue b/web/src/views/RepoAdd.vue index bef44379b8..6165eeb9f9 100644 --- a/web/src/views/RepoAdd.vue +++ b/web/src/views/RepoAdd.vue @@ -4,10 +4,6 @@ {{ $t('repo.add') }} - -
{ - repos.value = undefined; - repos.value = await apiClient.getRepoList({ all: true, flush: true }); - notifications.notify({ title: i18n.t('repo.enable.list_reloaded'), type: 'success' }); - }); - const { doSubmit: activateRepo, isLoading: isActivatingRepo } = useAsyncAction(async (repo: Repo) => { repoToActivate.value = repo; await apiClient.activateRepo(repo.owner, repo.name); @@ -85,11 +75,9 @@ export default defineComponent({ const goBack = useRouteBackOrDefault({ name: 'repos' }); return { - isReloadingRepos, isActivatingRepo, repoToActivate, goBack, - reloadRepos, activateRepo, searchedRepos, search, From 9254388f4a6d6fb4df5239e4466f5f92d86c8708 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Mon, 20 Mar 2023 17:59:52 +0100 Subject: [PATCH 02/12] Remove `RepoBatch` --- server/store/datastore/repo.go | 71 --------------------- server/store/datastore/repo_test.go | 99 ----------------------------- server/store/store.go | 2 - 3 files changed, 172 deletions(-) diff --git a/server/store/datastore/repo.go b/server/store/datastore/repo.go index df7b4ebf5e..5a80b85949 100644 --- a/server/store/datastore/repo.go +++ b/server/store/datastore/repo.go @@ -18,7 +18,6 @@ import ( "errors" "strings" - "github.com/rs/zerolog/log" "xorm.io/builder" "xorm.io/xorm" @@ -153,73 +152,3 @@ func (s storage) RepoList(user *model.User, owned bool) ([]*model.Repo, error) { Asc("repo_full_name"). Find(&repos) } - -// RepoBatch Sync batch of repos from SCM (with permissions) to store (create if not exist else update) -func (s storage) RepoBatch(repos []*model.Repo) error { - sess := s.engine.NewSession() - defer sess.Close() - if err := sess.Begin(); err != nil { - return err - } - - for i := range repos { - if len(repos[i].Owner) == 0 || len(repos[i].Name) == 0 || len(repos[i].FullName) == 0 { - log.Debug().Msgf("skip insert/update repo: %#v", repos[i]) - continue - } - - exist := true - repo, err := s.getRepoNameFallback(sess, repos[i].ForgeRemoteID, repos[i].FullName) - if err != nil { - if errors.Is(err, types.RecordNotExist) { - exist = false - } else { - return err - } - } - - // If it exists, just update. If it does not exist, it is not active, so we don't store it. - if exist { - if repos[i].FullName != repo.FullName { - // create redirection - err := s.createRedirection(sess, &model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) - if err != nil { - return err - } - } - if repos[i].ForgeRemoteID.IsValid() { - if _, err := sess. - Where("forge_remote_id = ?", repos[i].ForgeRemoteID). - Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_remote_id"). - Update(repos[i]); err != nil { - return err - } - } else { - if _, err := sess. - Where("repo_owner = ?", repos[i].Owner). - And(" repo_name = ?", repos[i].Name). - Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_remote_id"). - Update(repos[i]); err != nil { - return err - } - } - - _, err := sess. - Where("forge_remote_id = ?", repos[i].ForgeRemoteID). - Get(repos[i]) - if err != nil { - return err - } - } - - if repos[i].Perm != nil { - repos[i].Perm.RepoID = repos[i].ID - repos[i].Perm.Repo = repos[i] - if err := s.permUpsert(sess, repos[i].Perm); err != nil { - return err - } - } - } - - return sess.Commit() -} diff --git a/server/store/datastore/repo_test.go b/server/store/datastore/repo_test.go index 66e0309dd0..a20c1a3aa3 100644 --- a/server/store/datastore/repo_test.go +++ b/server/store/datastore/repo_test.go @@ -17,7 +17,6 @@ package datastore import ( "testing" - "time" "github.com/franela/goblin" "github.com/stretchr/testify/assert" @@ -293,104 +292,6 @@ func TestRepoCount(t *testing.T) { } } -func TestRepoBatch(t *testing.T) { - store, closer := newTestStore(t, new(model.Repo), new(model.User), new(model.Perm), new(model.Redirection)) - defer closer() - - if !assert.NoError(t, store.CreateRepo(&model.Repo{ - ForgeRemoteID: "5", - UserID: 1, - FullName: "foo/bar", - Owner: "foo", - Name: "bar", - IsActive: true, - })) { - return - } - - repos := []*model.Repo{ - { - ForgeRemoteID: "5", - UserID: 1, - FullName: "foo/bar", - Owner: "foo", - Name: "bar", - IsActive: true, - Perm: &model.Perm{ - UserID: 1, - Pull: true, - Push: true, - Admin: true, - Synced: time.Now().Unix(), - }, - }, - { - ForgeRemoteID: "6", - UserID: 1, - FullName: "bar/baz", - Owner: "bar", - Name: "baz", - IsActive: true, - }, - { - ForgeRemoteID: "7", - UserID: 1, - FullName: "baz/qux", - Owner: "baz", - Name: "qux", - IsActive: true, - }, - { - ForgeRemoteID: "8", - UserID: 0, // not activated repos do hot have a user id assigned - FullName: "baz/notes", - Owner: "baz", - Name: "notes", - IsActive: false, - }, - } - if !assert.NoError(t, store.RepoBatch(repos)) { - return - } - - // check DB state - perm, err := store.PermFind(&model.User{ID: 1}, repos[0]) - assert.NoError(t, err) - assert.True(t, perm.Admin) - - repo := &model.Repo{ - ForgeRemoteID: "5", - FullName: "foo/bar", - Owner: "foo", - Name: "bar", - Perm: &model.Perm{ - UserID: 1, - Pull: true, - Push: true, - Admin: false, - Synced: time.Now().Unix(), - }, - } - assert.NoError(t, store.RepoBatch([]*model.Repo{repo})) - assert.EqualValues(t, repos[0].ID, repo.ID) - - // check current DB state - _, err = store.engine.ID(repo.ID).Get(repo) - assert.NoError(t, err) - assert.True(t, repo.IsActive) - perm, err = store.PermFind(&model.User{ID: 1}, repos[0]) - assert.NoError(t, err) - assert.False(t, perm.Admin) - - allRepos := make([]*model.Repo, 0, 4) - assert.NoError(t, store.engine.Find(&allRepos)) - assert.Len(t, allRepos, 4) - - count, err := store.GetRepoCount() - assert.NoError(t, err) - assert.EqualValues(t, 3, count) -} - func TestRepoCrud(t *testing.T) { store, closer := newTestStore(t, new(model.Repo), diff --git a/server/store/store.go b/server/store/store.go index 7deb66f2d4..2a0c71298a 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -103,8 +103,6 @@ type Store interface { // RepoList TODO: paginate RepoList(user *model.User, owned bool) ([]*model.Repo, error) RepoListLatest(*model.User) ([]*model.Feed, error) - // RepoBatch Sync batch of repos from SCM (with permissions) to store (create if not exist else update) - RepoBatch([]*model.Repo) error // Permissions PermFind(user *model.User, repo *model.Repo) (*model.Perm, error) From 3eb61e299648906242d019c92f42246ef62d1016 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Mon, 20 Mar 2023 18:01:17 +0100 Subject: [PATCH 03/12] Fix badge creation --- server/api/badge.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/badge.go b/server/api/badge.go index 08cb91f864..8d8bfc0a94 100644 --- a/server/api/badge.go +++ b/server/api/badge.go @@ -36,8 +36,8 @@ import ( func GetBadge(c *gin.Context) { _store := store.FromContext(c) repo, err := _store.GetRepoName(c.Param("owner") + "/" + c.Param("name")) - if err != nil { - if errors.Is(err, types.RecordNotExist) { + if err != nil || !repo.IsActive { + if err == nil || errors.Is(err, types.RecordNotExist) { c.AbortWithStatus(http.StatusNotFound) return } From d78cb0ab7a3de2f1575d7e93e41b475d4513b817 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Mon, 20 Mar 2023 18:11:40 +0100 Subject: [PATCH 04/12] fix lint --- server/forge/bitbucket/bitbucket_test.go | 46 ------------------------ server/forge/gitlab/gitlab_test.go | 29 --------------- server/store/datastore/repo_test.go | 6 +++- server/store/mocks/store.go | 14 -------- 4 files changed, 5 insertions(+), 90 deletions(-) diff --git a/server/forge/bitbucket/bitbucket_test.go b/server/forge/bitbucket/bitbucket_test.go index 85cc7e96e4..031fc441d6 100644 --- a/server/forge/bitbucket/bitbucket_test.go +++ b/server/forge/bitbucket/bitbucket_test.go @@ -137,34 +137,6 @@ func Test_bitbucket(t *testing.T) { }) }) - g.Describe("When requesting repository permissions", func() { - g.It("Should handle not found errors", func() { - _, err := c.Perm(ctx, fakeUser, fakeRepoNotFound) - g.Assert(err).IsNotNil() - }) - g.It("Should authorize read access", func() { - perm, err := c.Perm(ctx, fakeUser, fakeRepoReadOnly) - g.Assert(err).IsNil() - g.Assert(perm.Pull).IsTrue() - g.Assert(perm.Push).IsFalse() - g.Assert(perm.Admin).IsFalse() - }) - g.It("Should authorize write access", func() { - perm, err := c.Perm(ctx, fakeUser, fakeRepoWriteOnly) - g.Assert(err).IsNil() - g.Assert(perm.Pull).IsTrue() - g.Assert(perm.Push).IsTrue() - g.Assert(perm.Admin).IsFalse() - }) - g.It("Should authorize admin access", func() { - perm, err := c.Perm(ctx, fakeUser, fakeRepoAdmin) - g.Assert(err).IsNil() - g.Assert(perm.Pull).IsTrue() - g.Assert(perm.Push).IsTrue() - g.Assert(perm.Admin).IsTrue() - }) - }) - g.Describe("When requesting user repositories", func() { g.It("Should return the details", func() { repos, err := c.Repos(ctx, fakeUser) @@ -333,24 +305,6 @@ var ( FullName: "test_name/hook_empty", } - fakeRepoReadOnly = &model.Repo{ - Owner: "test_name", - Name: "permission_read", - FullName: "test_name/permission_read", - } - - fakeRepoWriteOnly = &model.Repo{ - Owner: "test_name", - Name: "permission_write", - FullName: "test_name/permission_write", - } - - fakeRepoAdmin = &model.Repo{ - Owner: "test_name", - Name: "permission_admin", - FullName: "test_name/permission_admin", - } - fakePipeline = &model.Pipeline{ Commit: "9ecad50", } diff --git a/server/forge/gitlab/gitlab_test.go b/server/forge/gitlab/gitlab_test.go index c23f1711d2..28666884ff 100644 --- a/server/forge/gitlab/gitlab_test.go +++ b/server/forge/gitlab/gitlab_test.go @@ -107,35 +107,6 @@ func Test_GitLab(t *testing.T) { }) }) - // Test permissions method - g.Describe("Perm", func() { - g.It("Should return repo permissions", func() { - perm, err := client.Perm(ctx, &user, &repo) - assert.NoError(t, err) - assert.True(t, perm.Admin) - assert.True(t, perm.Pull) - assert.True(t, perm.Push) - }) - g.It("Should return repo permissions when user is admin", func() { - perm, err := client.Perm(ctx, &user, &model.Repo{ - Owner: "brightbox", - Name: "puppet", - }) - assert.NoError(t, err) - g.Assert(perm.Admin).Equal(true) - g.Assert(perm.Pull).Equal(true) - g.Assert(perm.Push).Equal(true) - }) - g.It("Should return error, when repo is not exist", func() { - _, err := client.Perm(ctx, &user, &model.Repo{ - Owner: "not-existed", - Name: "not-existed", - }) - - g.Assert(err).IsNotNil() - }) - }) - // Test activate method g.Describe("Activate", func() { g.It("Should be success", func() { diff --git a/server/store/datastore/repo_test.go b/server/store/datastore/repo_test.go index a20c1a3aa3..136cc9a43b 100644 --- a/server/store/datastore/repo_test.go +++ b/server/store/datastore/repo_test.go @@ -375,7 +375,11 @@ func TestRepoRedirection(t *testing.T) { Name: "test-renamed", } - assert.NoError(t, store.RepoBatch([]*model.Repo{&repoUpdated})) + assert.NoError(t, store.UpdateRepo(&repoUpdated)) + assert.NoError(t, store.CreateRedirection(&model.Redirection{ + RepoID: repo.ID, + FullName: repo.FullName, + })) // test redirection from old repo name repoFromStore, err := store.GetRepoNameFallback("1", "bradrydzewski/test") diff --git a/server/store/mocks/store.go b/server/store/mocks/store.go index 32dba86b9f..8190d08c9e 100644 --- a/server/store/mocks/store.go +++ b/server/store/mocks/store.go @@ -1451,20 +1451,6 @@ func (_m *Store) RegistryUpdate(_a0 *model.Registry) error { return r0 } -// RepoBatch provides a mock function with given fields: _a0 -func (_m *Store) RepoBatch(_a0 []*model.Repo) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func([]*model.Repo) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // RepoList provides a mock function with given fields: user, owned func (_m *Store) RepoList(user *model.User, owned bool) ([]*model.Repo, error) { ret := _m.Called(user, owned) From 92974b6b8a9ea1059723719d4888ab360250976b Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Mon, 20 Mar 2023 18:35:13 +0100 Subject: [PATCH 05/12] Fix nil pointers --- server/forge/gitea/fixtures/hooks.go | 28 +++++++++++++++--- server/forge/gitea/helper_test.go | 2 ++ server/forge/gitlab/testdata/projects.go | 36 ++++++++++++++++++++++-- server/forge/gogs/fixtures/hooks.go | 21 ++++++++++++-- server/forge/gogs/helper_test.go | 2 ++ server/store/datastore/repo_test.go | 1 + 6 files changed, 80 insertions(+), 10 deletions(-) diff --git a/server/forge/gitea/fixtures/hooks.go b/server/forge/gitea/fixtures/hooks.go index 8762c1305f..3890b0214e 100644 --- a/server/forge/gitea/fixtures/hooks.go +++ b/server/forge/gitea/fixtures/hooks.go @@ -52,7 +52,12 @@ const HookPush = ` "login": "gordon", "username": "gordon" }, - "private": true + "private": true, + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "pusher": { "name": "gordon", @@ -122,7 +127,12 @@ const HookPushBranch = ` "followers_count": 0, "following_count": 0, "starred_repos_count": 0, - "username": "meisam" + "username": "meisam", + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "name": "woodpecktester", "full_name": "meisam/woodpecktester", @@ -245,7 +255,12 @@ const HookPushTag = `{ "clone_url": "http://gitea.golang.org/gordon/hello-world.git", "default_branch": "master", "created_at": "2015-10-22T19:32:44Z", - "updated_at": "2016-11-24T13:37:16Z" + "updated_at": "2016-11-24T13:37:16Z", + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "sender": { "id": 1, @@ -300,7 +315,12 @@ const HookPullRequest = `{ "private": true, "html_url": "http://gitea.golang.org/gordon/hello-world", "clone_url": "https://gitea.golang.org/gordon/hello-world.git", - "default_branch": "master" + "default_branch": "master", + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "sender": { "id": 1, diff --git a/server/forge/gitea/helper_test.go b/server/forge/gitea/helper_test.go index 73cae85400..47f749cb8c 100644 --- a/server/forge/gitea/helper_test.go +++ b/server/forge/gitea/helper_test.go @@ -202,6 +202,7 @@ func Test_parse(t *testing.T) { HTMLURL: "http://gitea.golang.org/gophers/hello-world", Private: true, DefaultBranch: "master", + Permissions: &gitea.Permission{Admin: true}, } repo := toRepo(&from) g.Assert(repo.FullName).Equal(from.FullName) @@ -212,6 +213,7 @@ func Test_parse(t *testing.T) { g.Assert(repo.Clone).Equal(from.CloneURL) g.Assert(repo.Avatar).Equal(from.Owner.AvatarURL) g.Assert(repo.IsSCMPrivate).Equal(from.Private) + g.Assert(repo.Perm.Admin).IsTrue() }) g.It("Should correct a malformed avatar url", func() { diff --git a/server/forge/gitlab/testdata/projects.go b/server/forge/gitlab/testdata/projects.go index 482ae9e5b4..c64468a4c5 100644 --- a/server/forge/gitlab/testdata/projects.go +++ b/server/forge/gitlab/testdata/projects.go @@ -51,7 +51,17 @@ var allProjectsPayload = []byte(` "path": "diaspora", "updated_at": "2013-09-30T13:46:02Z" }, - "archived": false + "archived": false, + "permissions": { + "project_access": { + "access_level": 10, + "notification_level": 3 + }, + "group_access": { + "access_level": 50, + "notification_level": 3 + } + } }, { "id": 6, @@ -87,7 +97,17 @@ var allProjectsPayload = []byte(` "path": "brightbox", "updated_at": "2013-09-30T13:46:02Z" }, - "archived": true + "archived": true, + "permissions": { + "project_access": { + "access_level": 10, + "notification_level": 3 + }, + "group_access": { + "access_level": 50, + "notification_level": 3 + } + } } ] `) @@ -128,7 +148,17 @@ var notArchivedProjectsPayload = []byte(` "path": "diaspora", "updated_at": "2013-09-30T13:46:02Z" }, - "archived": false + "archived": false, + "permissions": { + "project_access": { + "access_level": 10, + "notification_level": 3 + }, + "group_access": { + "access_level": 50, + "notification_level": 3 + } + } } ] `) diff --git a/server/forge/gogs/fixtures/hooks.go b/server/forge/gogs/fixtures/hooks.go index cf2a542122..2f5ac87fed 100644 --- a/server/forge/gogs/fixtures/hooks.go +++ b/server/forge/gogs/fixtures/hooks.go @@ -48,7 +48,12 @@ var HookPush = ` "email": "gordon@golang.org", "username": "gordon" }, - "private": true + "private": true, + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "pusher": { "name": "gordon", @@ -88,7 +93,12 @@ var HookPushTag = `{ "clone_url": "http://gogs.golang.org/gordon/hello-world.git", "default_branch": "master", "created_at": "2015-10-22T19:32:44Z", - "updated_at": "2016-11-24T13:37:16Z" + "updated_at": "2016-11-24T13:37:16Z", + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "sender": { "id": 1, @@ -142,7 +152,12 @@ var HookPullRequest = `{ "private": true, "html_url": "http://gogs.golang.org/gordon/hello-world", "clone_url": "https://gogs.golang.org/gordon/hello-world.git", - "default_branch": "master" + "default_branch": "master", + "permissions": { + "admin": true, + "push": true, + "pull": true + } }, "sender": { "id": 1, diff --git a/server/forge/gogs/helper_test.go b/server/forge/gogs/helper_test.go index df80081e66..70a53daf86 100644 --- a/server/forge/gogs/helper_test.go +++ b/server/forge/gogs/helper_test.go @@ -174,6 +174,7 @@ func Test_parse(t *testing.T) { HTMLURL: "http://gogs.golang.org/gophers/hello-world", Private: true, DefaultBranch: "master", + Permissions: &gogs.Permission{Admin: true}, } repo := toRepo(&from, false) g.Assert(repo.FullName).Equal(from.FullName) @@ -184,6 +185,7 @@ func Test_parse(t *testing.T) { g.Assert(repo.Clone).Equal(from.CloneURL) g.Assert(repo.Avatar).Equal(from.Owner.AvatarUrl) g.Assert(repo.IsSCMPrivate).Equal(from.Private) + g.Assert(repo.Perm.Admin).IsTrue() }) g.It("Should correct a malformed avatar url", func() { diff --git a/server/store/datastore/repo_test.go b/server/store/datastore/repo_test.go index 136cc9a43b..feedf75b52 100644 --- a/server/store/datastore/repo_test.go +++ b/server/store/datastore/repo_test.go @@ -369,6 +369,7 @@ func TestRepoRedirection(t *testing.T) { assert.NoError(t, store.CreateRepo(&repo)) repoUpdated := model.Repo{ + ID: repo.ID, ForgeRemoteID: "1", FullName: "bradrydzewski/test-renamed", Owner: "bradrydzewski", From b239c1c811d4d2388fb13d525d16650d2c46d61e Mon Sep 17 00:00:00 2001 From: qwerty287 <80460567+qwerty287@users.noreply.github.com> Date: Tue, 21 Mar 2023 12:56:58 +0100 Subject: [PATCH 06/12] Apply suggestions from code review Co-authored-by: Anbraten --- server/api/repo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/repo.go b/server/api/repo.go index a0a550b855..e72ae1eb93 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -44,7 +44,7 @@ func PostRepo(c *gin.Context) { name := c.Param("name") repo, err := _store.GetRepoName(owner + "/" + name) enabledOnce := err == nil // if there's no error, the repo was found and enabled once already - if err == nil && repo.IsActive { + if enabledOnce && repo.IsActive { c.String(http.StatusConflict, "Repository is already active.") return } else if err != nil && !errors.Is(err, types.RecordNotExist) { @@ -58,7 +58,7 @@ func PostRepo(c *gin.Context) { return } if !from.Perm.Admin { - c.String(http.StatusForbidden, "User not authorized") + c.String(http.StatusForbidden, "User has to be a admin of this repository") } if enabledOnce { From f5dab64b261e07767051b57f349ef982273a775a Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Tue, 21 Mar 2023 12:59:06 +0100 Subject: [PATCH 07/12] group import --- server/api/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/repo.go b/server/api/repo.go index afae7728de..a2bfaf7a0d 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -26,12 +26,12 @@ import ( "github.com/gin-gonic/gin" "github.com/gorilla/securecookie" "github.com/rs/zerolog/log" - "github.com/woodpecker-ci/woodpecker/server/store/types" "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server/store" + "github.com/woodpecker-ci/woodpecker/server/store/types" "github.com/woodpecker-ci/woodpecker/shared/token" ) From 46519e9fec318af7f9023851b2c5377c761fcaad Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Tue, 21 Mar 2023 12:59:22 +0100 Subject: [PATCH 08/12] rm old code --- server/api/user.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/server/api/user.go b/server/api/user.go index 0070fc3cc6..84d57746cb 100644 --- a/server/api/user.go +++ b/server/api/user.go @@ -63,30 +63,6 @@ func GetRepos(c *gin.Context) { user := session.User(c) all, _ := strconv.ParseBool(c.Query("all")) - /*if flush || time.Unix(user.Synced, 0).Add(time.Hour*72).Before(time.Now()) { - log.Debug().Msgf("sync begin: %s", user.Login) - user.Synced = time.Now().Unix() - if err := _store.UpdateUser(user); err != nil { - log.Err(err).Msgf("update user '%s'", user.Login) - return - } - - config := ToConfig(c) - - sync := forge.Syncer{ - Forge: _forge, - Store: _store, - Perms: _store, - Match: forge.NamespaceFilter(config.OwnersWhitelist), - } - - if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil { - log.Debug().Msgf("sync error: %s: %s", user.Login, err) - } else { - log.Debug().Msgf("sync complete: %s", user.Login) - } - }*/ - dbRepos, err := _store.RepoList(user, true) if err != nil { c.String(http.StatusInternalServerError, "Error fetching repository list. %s", err) From 4e16b8fb2c5e45997c238e0142ad133bf343358c Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Tue, 21 Mar 2023 15:16:06 +0100 Subject: [PATCH 09/12] Remove flat permission option as it's not used anymore --- cmd/server/flags.go | 10 ---------- cmd/server/server.go | 3 --- server/config.go | 1 - 3 files changed, 14 deletions(-) diff --git a/cmd/server/flags.go b/cmd/server/flags.go index c6d96f8d1b..de13170018 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -487,16 +487,6 @@ var flags = []cli.Flag{ Hidden: true, }, // - // misc - // - &cli.BoolFlag{ - EnvVars: []string{"WOODPECKER_FLAT_PERMISSIONS"}, - Name: "flat-permissions", - Usage: "no forge call for permissions should be made", - Hidden: true, - // TODO(485) temporary workaround to not hit api rate limits - }, - // // secrets encryption in DB // &cli.StringFlag{ diff --git a/cmd/server/server.go b/cmd/server/server.go index 965d524fb2..cf3a99af16 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -360,7 +360,4 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) { // prometheus server.Config.Prometheus.AuthToken = c.String("prometheus-auth-token") - - // TODO(485) temporary workaround to not hit api rate limits - server.Config.FlatPermissions = c.Bool("flat-permissions") } diff --git a/server/config.go b/server/config.go index 113e4f9549..8633147862 100644 --- a/server/config.go +++ b/server/config.go @@ -84,5 +84,4 @@ var Config = struct { DefaultTimeout int64 MaxTimeout int64 } - FlatPermissions bool // TODO(485) temporary workaround to not hit api rate limits }{} From 4b2e681a81e92d3c64fdce026a6f1ae167b56d23 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Tue, 21 Mar 2023 16:34:23 +0100 Subject: [PATCH 10/12] Fix deletion of repos that were active once --- server/store/datastore/migration/015_remove_inactive_repos.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/store/datastore/migration/015_remove_inactive_repos.go b/server/store/datastore/migration/015_remove_inactive_repos.go index 76491b6912..d042ea7ea8 100644 --- a/server/store/datastore/migration/015_remove_inactive_repos.go +++ b/server/store/datastore/migration/015_remove_inactive_repos.go @@ -22,7 +22,8 @@ var removeInactiveRepos = task{ name: "remove-inactive-repos", required: true, fn: func(sess *xorm.Session) error { - _, err := sess.Table("repos").Where("repo_active = ?", false).Delete() + // If the timeout is 0, the repo was never activated, so we remove it. + _, err := sess.Table("repos").Where("repo_active = ?", false).Where("repo_timeout != ?", 0).Delete() if err != nil { return err } From da4593ada625dd9caf280b64322e0377518cdee2 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Tue, 21 Mar 2023 16:56:14 +0100 Subject: [PATCH 11/12] fix migration --- server/store/datastore/migration/015_remove_inactive_repos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/store/datastore/migration/015_remove_inactive_repos.go b/server/store/datastore/migration/015_remove_inactive_repos.go index d042ea7ea8..1c6de6b296 100644 --- a/server/store/datastore/migration/015_remove_inactive_repos.go +++ b/server/store/datastore/migration/015_remove_inactive_repos.go @@ -23,7 +23,7 @@ var removeInactiveRepos = task{ required: true, fn: func(sess *xorm.Session) error { // If the timeout is 0, the repo was never activated, so we remove it. - _, err := sess.Table("repos").Where("repo_active = ?", false).Where("repo_timeout != ?", 0).Delete() + _, err := sess.Table("repos").Where("repo_active = ?", false).And("repo_timeout = ?", 0).Delete() if err != nil { return err } From 0e796472a235067407d2be16984f9b296bfb058b Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Tue, 21 Mar 2023 18:58:35 +0100 Subject: [PATCH 12/12] Explicitely set new perms on repair --- server/api/repo.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/repo.go b/server/api/repo.go index a2bfaf7a0d..64db2f8007 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -318,7 +318,9 @@ func RepairRepo(c *gin.Context) { _ = c.AbortWithError(http.StatusInternalServerError, err) return } - repo.Perm = from.Perm + repo.Perm.Pull = from.Perm.Pull + repo.Perm.Push = from.Perm.Push + repo.Perm.Admin = from.Perm.Admin if err := _store.PermUpsert(repo.Perm); err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) return