From 6e22c5430f1c801b397130857f9a3b90c90c21d6 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sat, 7 Sep 2024 19:35:16 +0530 Subject: [PATCH 01/10] refactoring the routes and adding groups to the router --- .gitignore | 1 + internal/config/config.go | 34 +++++++++++++++++ internal/server/middleware.go | 28 ++++++++++++++ internal/server/profiling.go | 30 +++++++++++++++ internal/server/router.go | 66 +++++++++++++++++++++++++++++++++ internal/server/router_group.go | 36 ++++++++++++++++++ internal/server/server.go | 42 +++++++++++++++++++++ main.go | 54 +++++++++++++++------------ 8 files changed, 268 insertions(+), 23 deletions(-) create mode 100644 internal/config/config.go create mode 100644 internal/server/middleware.go create mode 100644 internal/server/profiling.go create mode 100644 internal/server/router.go create mode 100644 internal/server/router_group.go create mode 100644 internal/server/server.go diff --git a/.gitignore b/.gitignore index a5581a5..1f90f92 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ dist/ .DS_Store zot/cache.db secrets.txt +__debug_bin1949266242 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..0f0da6e --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,34 @@ +package config + +import ( + "fmt" + + "github.com/spf13/viper" +) + +type Config struct { + LogLevel string + OwnRegistry bool + OwnRegistryAdr string + OwnRegistryPort string + ZotConfigPath string + Input string +} + +func NewConfig() (*Config, error) { + viper.SetConfigName("config") + viper.SetConfigType("toml") + viper.AddConfigPath(".") + if err := viper.ReadInConfig(); err != nil { + return nil, fmt.Errorf("error reading config file: %w", err) + } + + return &Config{ + LogLevel: viper.GetString("log_level"), + OwnRegistry: viper.GetBool("bring_own_registry"), + OwnRegistryAdr: viper.GetString("own_registry_adr"), + OwnRegistryPort: viper.GetString("own_registry_port"), + ZotConfigPath: viper.GetString("zotConfigPath"), + Input: viper.GetString("url_or_file"), + }, nil +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go new file mode 100644 index 0000000..d320124 --- /dev/null +++ b/internal/server/middleware.go @@ -0,0 +1,28 @@ +package server + +import ( + "log" + "net/http" + "time" +) + +type Middleware func(http.Handler) http.Handler + +// LoggingMiddleware creates a new logging middleware +func LoggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + // Call the next handler + next.ServeHTTP(w, r) + + // Log the request + log.Printf( + "%s %s %s %s", + r.RemoteAddr, + r.Method, + r.URL.Path, + time.Since(start), + ) + }) +} diff --git a/internal/server/profiling.go b/internal/server/profiling.go new file mode 100644 index 0000000..fd89d02 --- /dev/null +++ b/internal/server/profiling.go @@ -0,0 +1,30 @@ +package server + +import ( + "net/http/pprof" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type MetricsRegistrar struct{} + +type DebugRegistrar struct{} + +func (m *MetricsRegistrar) RegisterRoutes(router Router) { + metricsGroup := router.Group("/metrics") + metricsGroup.Handle("", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) +} + +func (dr *DebugRegistrar) RegisterRoutes(router Router) { + debugGroup := router.Group("/debug/pprof") + debugGroup.HandleFunc("/", pprof.Index) + debugGroup.HandleFunc("/profile", pprof.Profile) + debugGroup.HandleFunc("/trace", pprof.Trace) + debugGroup.HandleFunc("/symbol", pprof.Symbol) + debugGroup.Handle("/heap", pprof.Handler("heap")) + debugGroup.Handle("/goroutine", pprof.Handler("goroutine")) + debugGroup.Handle("/threadcreate", pprof.Handler("threadcreate")) + debugGroup.Handle("/block", pprof.Handler("block")) + debugGroup.Handle("/mutex", pprof.Handler("mutex")) +} diff --git a/internal/server/router.go b/internal/server/router.go new file mode 100644 index 0000000..4c2a4c2 --- /dev/null +++ b/internal/server/router.go @@ -0,0 +1,66 @@ +package server + +import "net/http" + +type Router interface { + // Handle registers a new route with the given pattern and handler on the router mux + Handle(pattern string, handler http.Handler) + // HandleFunc registers a new route with the given pattern and handler function on the router mux + HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) + // ServeHTTP would dispatch the request to the default mux + ServeHTTP(w http.ResponseWriter, r *http.Request) + // Use adds middleware to the default router + Use(middleware ...Middleware) + // Group called on the router would create a group with the given prefix + //and would inherit the middleware from the router and would be added to the root group of the router + Group(prefix string) *RouterGroup +} + +type Endpoint struct { + Method string + Path string +} +// DefaultRouter +type DefaultRouter struct { + // mux is the default http.ServeMux + mux *http.ServeMux + // middleware is the list of middleware to be applied to default router and all groups inside it + middleware []Middleware + // rootGroup is the root RouterGroup + rootGroup *RouterGroup + // endpoints is the list of all registered endpoints + Endpoints []Endpoint +} + +// NewDefaultRouter creates a new DefaultRouter with the given prefix +func NewDefaultRouter(prefix string) *DefaultRouter { + dr := &DefaultRouter{mux: http.NewServeMux()} + dr.rootGroup = &RouterGroup{prefix: prefix, router: dr} + return dr +} + + +func (r *DefaultRouter) Handle(pattern string, handler http.Handler) { + r.mux.Handle(pattern, handler) +} + + +func (r *DefaultRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + r.mux.ServeHTTP(w, req) +} + + +func (dr *DefaultRouter) HandleFunc(pattern string, handlerFunc func(http.ResponseWriter, *http.Request)) { + dr.Handle(pattern, http.HandlerFunc(handlerFunc)) +} + + +func (dr *DefaultRouter) Use(middleware ...Middleware) { + dr.middleware = append(dr.middleware, middleware...) +} + + +// Group creates a new RouterGroup under the rootGroup with the given prefix +func (dr *DefaultRouter) Group(prefix string) *RouterGroup { + return dr.rootGroup.Group(prefix) +} diff --git a/internal/server/router_group.go b/internal/server/router_group.go new file mode 100644 index 0000000..8964241 --- /dev/null +++ b/internal/server/router_group.go @@ -0,0 +1,36 @@ + +package server + +import "net/http" + +// RouterGroup represents a group of routes with a common prefix and middleware +type RouterGroup struct { + prefix string + middleware []Middleware + router Router +} + +func (rg *RouterGroup) Use(middleware ...Middleware) { + rg.middleware = append(rg.middleware, middleware...) +} + +func (rg *RouterGroup) Handle(pattern string, handler http.Handler) { + fullPattern := rg.prefix + pattern + wrappedHandler := handler + for i := len(rg.middleware) - 1; i >= 0; i-- { + wrappedHandler = rg.middleware[i](wrappedHandler) + } + rg.router.Handle(fullPattern, wrappedHandler) +} + +func (rg *RouterGroup) HandleFunc(pattern string, handlerFunc func(http.ResponseWriter, *http.Request)) { + rg.Handle(pattern, http.HandlerFunc(handlerFunc)) +} + +func (rg *RouterGroup) Group(prefix string) *RouterGroup { + return &RouterGroup{ + prefix: rg.prefix + prefix, + middleware: append([]Middleware{}, rg.middleware...), + router: rg.router, + } +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..3f83444 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,42 @@ +package server + +import ( + "context" + "net/http" +) + +type RouteRegistrar interface { + RegisterRoutes(router Router) +} + +// App struct with middleware support +type App struct { + router Router + registrars []RouteRegistrar + server *http.Server +} + +func NewApp(router Router, registrars ...RouteRegistrar) *App { + return &App{ + router: router, + registrars: registrars, + server: &http.Server{Addr: ":9090", Handler: router}, + } +} + +func (a *App) SetupRoutes() { + for _, registrar := range a.registrars { + registrar.RegisterRoutes(a.router) + } +} + +func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { + a.router.ServeHTTP(w, r) +} +func (a *App) Start() error { + return a.server.ListenAndServe() +} + +func (a *App) Shutdown(ctx context.Context) error { + return a.server.Shutdown(ctx) +} diff --git a/main.go b/main.go index 78ff515..e2be8b7 100644 --- a/main.go +++ b/main.go @@ -7,7 +7,6 @@ import ( "fmt" "net" "net/http" - "net/http/pprof" "net/url" "os" "os/signal" @@ -18,14 +17,12 @@ import ( "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" + "container-registry.com/harbor-satellite/internal/server" "container-registry.com/harbor-satellite/internal/store" "container-registry.com/harbor-satellite/logger" "container-registry.com/harbor-satellite/registry" "golang.org/x/sync/errgroup" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/spf13/viper" "github.com/joho/godotenv" @@ -70,32 +67,43 @@ func run() error { log := logger.FromContext(ctx) log.Info().Msg("Satellite starting") - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) - mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) - mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) - mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) - mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) - mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) - mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) - mux.Handle("/debug/pprof/block", pprof.Handler("block")) - mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) - metricsSrv := &http.Server{ - Addr: ":9090", - Handler: mux, - } + router := server.NewDefaultRouter("/api/v1") + router.Use(server.LoggingMiddleware) + + app := server.NewApp( + router, + &server.MetricsRegistrar{}, + &server.DebugRegistrar{}, + ) + + app.SetupRoutes() + // Start the server in a goroutine g.Go(func() error { - if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Println("Starting server on :9090") + if err := app.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("Server error: %v", err) return err } return nil }) + + // Graceful shutdown g.Go(func() error { <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - return metricsSrv.Shutdown(shutdownCtx) + log.Info().Msg("Shutdown signal received") + + // Create a timeout context for shutdown + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelShutdown() + + log.Info().Msg("Shutting down server...") + if err := app.Shutdown(shutdownCtx); err != nil { + log.Error().Err(err).Msg("Server shutdown error") + return err + } + + log.Info().Msg("Server gracefully stopped") + return nil }) bringOwnRegistry := viper.GetBool("bring_own_registry") From 88e8588a2218d53c2f5097469bc2731a4e2283cb Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 8 Sep 2024 16:15:13 +0530 Subject: [PATCH 02/10] simpligying the harbor config --- internal/config/config.go | 2 +- internal/server/server.go | 28 ++++++++++++++++++- logger/logger.go | 6 ++++ main.go | 58 ++++++++------------------------------- 4 files changed, 46 insertions(+), 48 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 0f0da6e..44331b6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,7 +15,7 @@ type Config struct { Input string } -func NewConfig() (*Config, error) { +func LoadConfig() (*Config, error) { viper.SetConfigName("config") viper.SetConfigType("toml") viper.AddConfigPath(".") diff --git a/internal/server/server.go b/internal/server/server.go index 3f83444..7da7785 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,7 +2,12 @@ package server import ( "context" + "errors" "net/http" + + "container-registry.com/harbor-satellite/internal/config" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" ) type RouteRegistrar interface { @@ -14,12 +19,18 @@ type App struct { router Router registrars []RouteRegistrar server *http.Server + ctx context.Context + Logger *zerolog.Logger + config *config.Config } -func NewApp(router Router, registrars ...RouteRegistrar) *App { +func NewApp(router Router, ctx context.Context, logger *zerolog.Logger, config *config.Config, registrars ...RouteRegistrar) *App { return &App{ router: router, registrars: registrars, + ctx: ctx, + Logger: logger, + config: config, server: &http.Server{Addr: ":9090", Handler: router}, } } @@ -40,3 +51,18 @@ func (a *App) Start() error { func (a *App) Shutdown(ctx context.Context) error { return a.server.Shutdown(ctx) } + +func (a *App) SetupServer(g *errgroup.Group) { + g.Go(func() error { + a.Logger.Info().Msg("Starting server on :9090") + if err := a.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }) + g.Go(func() error { + <-a.ctx.Done() + a.Logger.Info().Msg("Shutting down server") + return a.Shutdown(a.ctx) + }) +} diff --git a/logger/logger.go b/logger/logger.go index 78664dc..13c85af 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -48,3 +48,9 @@ func FromContext(ctx context.Context) *zerolog.Logger { } return logger } + +func SetupLogger(ctx context.Context, logLevel string) *zerolog.Logger{ + ctx = AddLoggerToContext(ctx, logLevel) + logger := FromContext(ctx) + return logger +} diff --git a/main.go b/main.go index e2be8b7..3bc7cac 100644 --- a/main.go +++ b/main.go @@ -6,15 +6,14 @@ import ( "errors" "fmt" "net" - "net/http" "net/url" "os" "os/signal" "path/filepath" "strings" "syscall" - "time" + "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/server" @@ -39,32 +38,24 @@ type ImageList struct { } func main() { - viper.SetConfigName("config") - viper.SetConfigType("toml") - viper.AddConfigPath(".") - if err := viper.ReadInConfig(); err != nil { - fmt.Println("Error reading config file, ", err) - fmt.Println("Exiting Satellite") - os.Exit(1) - } - - err := run() - if err != nil { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } } func run() error { + config, err := config.LoadConfig() + if err != nil { + return err + } var fetcher store.ImageFetcher ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM) defer cancel() g, ctx := errgroup.WithContext(ctx) - logLevel := viper.GetString("log_level") - ctx = logger.AddLoggerToContext(ctx, logLevel) - - log := logger.FromContext(ctx) + log := logger.SetupLogger(ctx, config.LogLevel) log.Info().Msg("Satellite starting") router := server.NewDefaultRouter("/api/v1") @@ -72,40 +63,15 @@ func run() error { app := server.NewApp( router, + ctx, + log, + config, &server.MetricsRegistrar{}, &server.DebugRegistrar{}, ) app.SetupRoutes() - // Start the server in a goroutine - g.Go(func() error { - log.Println("Starting server on :9090") - if err := app.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("Server error: %v", err) - return err - } - return nil - }) - - // Graceful shutdown - g.Go(func() error { - <-ctx.Done() - log.Info().Msg("Shutdown signal received") - - // Create a timeout context for shutdown - shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 30*time.Second) - defer cancelShutdown() - - log.Info().Msg("Shutting down server...") - if err := app.Shutdown(shutdownCtx); err != nil { - log.Error().Err(err).Msg("Server shutdown error") - return err - } - - log.Info().Msg("Server gracefully stopped") - return nil - }) - + app.SetupServer(g) bringOwnRegistry := viper.GetBool("bring_own_registry") if bringOwnRegistry { registryAdr := viper.GetString("own_registry_adr") From 7ed64c3a2d1e00ab018bef49a866c3014839c2bb Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Mon, 9 Sep 2024 02:59:36 +0530 Subject: [PATCH 03/10] fixing running test cases using dagger run command --- internal/satellite/routes.go | 34 +++++++++++++++++++++++++++++ main.go | 1 + test/e2e/satellite_test.go | 41 +++++++++++++++++++++++------------ test/e2e/testdata/config.toml | 24 +++++++++++++++----- 4 files changed, 80 insertions(+), 20 deletions(-) create mode 100644 internal/satellite/routes.go diff --git a/internal/satellite/routes.go b/internal/satellite/routes.go new file mode 100644 index 0000000..1f43fd2 --- /dev/null +++ b/internal/satellite/routes.go @@ -0,0 +1,34 @@ +package satellite + +import ( + "encoding/json" + "net/http" + + "container-registry.com/harbor-satellite/internal/server" +) + +type SatelliteRegistrar struct{} + +type SatelliteResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + StatusCode int `json:"status_code"` +} + +func (sr *SatelliteRegistrar) RegisterRoutes(router server.Router) { + satelliteGroup := router.Group("/satellite") + satelliteGroup.HandleFunc("/ping", sr.Ping) +} + +func (sr *SatelliteRegistrar) Ping(w http.ResponseWriter, r *http.Request) { + response := SatelliteResponse{ + Success: true, + Message: "Ping satellite successful", + StatusCode: http.StatusOK, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(response); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} diff --git a/main.go b/main.go index 3bc7cac..e5cb0aa 100644 --- a/main.go +++ b/main.go @@ -68,6 +68,7 @@ func run() error { config, &server.MetricsRegistrar{}, &server.DebugRegistrar{}, + &satellite.SatelliteRegistrar{}, ) app.SetupRoutes() diff --git a/test/e2e/satellite_test.go b/test/e2e/satellite_test.go index b923aae..8d69c39 100644 --- a/test/e2e/satellite_test.go +++ b/test/e2e/satellite_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -13,9 +14,12 @@ import ( ) const ( - appDir = "/app" - appBinary = "app" - sourceFile = "main.go" + appDir = "/app" + appBinary = "app" + sourceFile = "main.go" + relative_path = "./testdata/config.toml" + absolute_path = "test/e2e/testdata/config.toml" + satellite_ping_endpoint = "/api/v1/satellite/ping" ) func TestSatellite(t *testing.T) { @@ -118,7 +122,6 @@ func pushImageToSourceRegistry( fmt.Println(stdOut) } -// buildSatellite and test test the connection func buildSatellite( t *testing.T, client *dagger.Client, @@ -136,7 +139,7 @@ func buildSatellite( dir := client.Host().Directory(parentDir) // Get configuration file on the host - configFile := client.Host().File("./testdata/config.toml") + configFile := client.Host().File(absolute_path) // Configure and build the Satellite container := client.Container().From("golang:alpine").WithDirectory(appDir, dir). @@ -149,17 +152,27 @@ func buildSatellite( WithExec([]string{"cat", "config.toml"}). WithFile("./config.toml", configFile). WithExec([]string{"cat", "config.toml"}). - WithExec([]string{"apk", "add", "crane"}). - WithExec([]string{"crane", "-v", "catalog", "source:5000", "--insecure"}). - WithExec([]string{"crane", "digest", "source:5000/library/busybox:1.36", "--insecure"}). WithExec([]string{"go", "build", "-o", appBinary, sourceFile}). WithExposedPort(9090). - WithExec([]string{"go", "run", "./test/e2e/test.go"}) - - assert.NoError(t, err, "Test failed in buildSatellite") - - stdOut, _ := container.Stdout(ctx) - fmt.Println(stdOut) + WithExec([]string{"./" + appBinary}). + AsService() + // Get the response + satellite_container := client.Container(). + From("golang:alpine"). + WithServiceBinding("satellite", container). + WithExec([]string{"wget", "-O", "-", "http://satellite:9090" + satellite_ping_endpoint}) + + output, err := satellite_container.Stdout(ctx) + assert.NoError(t, err, "Failed to get output from satellite ping endpoint") + // Check the response + var response map[string]interface{} + err = json.Unmarshal([]byte(output), &response) + assert.NoError(t, err, "Failed to parse JSON response") + + // Assert the response + assert.Equal(t, true, response["success"], "Unexpected success value") + assert.Equal(t, "Ping satellite successful", response["message"], "Unexpected message") + assert.Equal(t, float64(200), response["status_code"], "Unexpected status code") } // Gets the directory of the project diff --git a/test/e2e/testdata/config.toml b/test/e2e/testdata/config.toml index 06d9736..d9ab5a1 100644 --- a/test/e2e/testdata/config.toml +++ b/test/e2e/testdata/config.toml @@ -1,7 +1,19 @@ -bring_own_registry = true -own_registry_adr = "dest:5000" -url_or_file = "http://source:5000/v2/library/busybox" +# Wether to us the built-in Zot registry or not +bring_own_registry = false -# Additional test cases need to be handled. -# url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" -# url_or_file = "http://localhost:5001/v2/library/busybox" +# IP address and port of own registry +own_registry_adr = "127.0.0.1" +own_registry_port = "8585" + +# URL of remote registry OR local file path +url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" + +# Default path for Zot registry config.json +zotConfigPath = "./registry/config.json" + +# Set logging level +log_level = "info" + +# For testing purposes : +# https://demo.goharbor.io/v2/myproject/album-server +# /image-list/images.json From 38071424cfac844323811354c934ea3fb3fe1511 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Wed, 11 Sep 2024 02:06:35 +0530 Subject: [PATCH 04/10] adding absolute path flag in test case to configure the path to the config file --- ci/README.md | 13 ++++++++-- test/e2e/config.go | 18 +++++++++++++ test/e2e/satellite_test.go | 20 ++++++-------- test/e2e/test.go | 53 -------------------------------------- 4 files changed, 37 insertions(+), 67 deletions(-) create mode 100644 test/e2e/config.go delete mode 100644 test/e2e/test.go diff --git a/ci/README.md b/ci/README.md index b6282c1..89095a1 100644 --- a/ci/README.md +++ b/ci/README.md @@ -61,14 +61,23 @@ To run a particular function, run: To build the satellite binaries, use the following command: - ```sh dagger call build --source=. --name=satellite export --path=./bin -This would spin up a container and install required dependencies and build various architecture binaries and export them to the host on path ./bin for testing on the host. + This would spin up a container and install required dependencies and build various architecture binaries and export them to the host on path ./bin for testing on the host. - #### Example: Releasing to GitHub To release the project on GitHub, use the following command - ```sh dagger call release --directory=. --token= --name=satellite -The above function would then proceed to release the project on github for the name provided. The above function also takes argument `--release-type` which would tell the release what kind of release it is i.e major, minor or path, The default value is set to be path release + The above function would then proceed to release the project on github for the name provided. The above function also takes argument `--release-type` which would tell the release what kind of release it is i.e major, minor or path, The default value is set to be path release - #### Example: Releasing to GitHub with type of release To release the project on GitHub, use the following command - ```sh dagger call release --directory=. --token= --name=satellite --release-type=minor The above function would release the minor version for the project mentioned +- #### Example: Running test cases using dagger + To run the test cases using dagger use the following command + - ```sh + dagger run go test ./... -v -count=1 + This would run the test cases present in the entire project. + To run the test cases without dagger use the following command + - ```sh + go test ./... -v -count=1 -args -abs=false + This would set the config file to use the relative path. diff --git a/test/e2e/config.go b/test/e2e/config.go new file mode 100644 index 0000000..1559086 --- /dev/null +++ b/test/e2e/config.go @@ -0,0 +1,18 @@ +package e2e + +import "flag" + +const ( + appDir = "/app" + appBinary = "app" + sourceFile = "main.go" + relative_path = "./testdata/config.toml" + absolute_path = "test/e2e/testdata/config.toml" + satellite_ping_endpoint = "/api/v1/satellite/ping" +) + +var ABS bool + +func init() { + flag.BoolVar(&ABS, "abs", true, "Use absolute path for the config file") +} diff --git a/test/e2e/satellite_test.go b/test/e2e/satellite_test.go index 8d69c39..9d7435f 100644 --- a/test/e2e/satellite_test.go +++ b/test/e2e/satellite_test.go @@ -1,4 +1,4 @@ -package main +package e2e import ( "context" @@ -13,15 +13,6 @@ import ( "github.com/stretchr/testify/assert" ) -const ( - appDir = "/app" - appBinary = "app" - sourceFile = "main.go" - relative_path = "./testdata/config.toml" - absolute_path = "test/e2e/testdata/config.toml" - satellite_ping_endpoint = "/api/v1/satellite/ping" -) - func TestSatellite(t *testing.T) { ctx := context.Background() @@ -130,7 +121,12 @@ func buildSatellite( dest *dagger.Service, ) { socket := client.Host().UnixSocket("/var/run/docker.sock") - + var PATH_TO_CONFIG string + if ABS { + PATH_TO_CONFIG = absolute_path + } else { + PATH_TO_CONFIG = relative_path + } // Get the directory parentDir, err := getProjectDir() assert.NoError(t, err, "Failed to get Project Directory") @@ -139,7 +135,7 @@ func buildSatellite( dir := client.Host().Directory(parentDir) // Get configuration file on the host - configFile := client.Host().File(absolute_path) + configFile := client.Host().File(PATH_TO_CONFIG) // Configure and build the Satellite container := client.Container().From("golang:alpine").WithDirectory(appDir, dir). diff --git a/test/e2e/test.go b/test/e2e/test.go deleted file mode 100644 index 53ad8f7..0000000 --- a/test/e2e/test.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "log" - "os" - "os/exec" - "strings" -) - -func main() { - // Command to execute - cmd := exec.Command("go", "run", "./main.go") - - // Get stdout pipe - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatalf("Error creating stdout pipe: %v", err) - } - - // Start the command - if err := cmd.Start(); err != nil { - log.Fatalf("Error starting command: %v", err) - } - - // Create a scanner to read the command output line by line - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - line := scanner.Text() - fmt.Println(line) // Print each line of output - - lineParts := strings.Split(line, "--") - // Check if the line contains "----" - if len(lineParts) > 2 { - fmt.Println("Satellite is Working...\nExiting...") - if err := cmd.Process.Kill(); err != nil { - fmt.Println("Error killing process:", err) - } - os.Exit(0) // Exit the program - } - } - - // Handle any scanner error - if err := scanner.Err(); err != nil { - log.Fatalf("Error reading stdout: %v", err) - } - - // Wait for the command to finish - if err := cmd.Wait(); err != nil { - log.Fatalf("Command execution failed: %v", err) - } -} From ce6a19e9b833945e31bf7578eef7a46ed3a639a2 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sat, 14 Sep 2024 00:40:33 +0530 Subject: [PATCH 05/10] adding utils for simplification --- internal/config/config.go | 122 +++++++++++++++-- internal/images/get-images.go | 11 ++ internal/utils/utils.go | 131 +++++++++++++++++++ main.go | 237 +++++++++++++++------------------- 4 files changed, 353 insertions(+), 148 deletions(-) create mode 100644 internal/utils/utils.go diff --git a/internal/config/config.go b/internal/config/config.go index 44331b6..8948647 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,13 +6,102 @@ import ( "github.com/spf13/viper" ) +var AppConfig *Config + type Config struct { - LogLevel string - OwnRegistry bool - OwnRegistryAdr string - OwnRegistryPort string - ZotConfigPath string - Input string + logLevel string + ownRegistry bool + ownRegistryAdr string + ownRegistryPort string + zotConfigPath string + input string + zot_url string + registry string + repository string + user_input string + scheme string + api_version string + image string +} + +func GetLogLevel() string { + return AppConfig.logLevel +} + +func GetOwnRegistry() bool { + return AppConfig.ownRegistry +} + +func GetOwnRegistryAdr() string { + return AppConfig.ownRegistryAdr +} + +func GetOwnRegistryPort() string { + return AppConfig.ownRegistryPort +} + +func GetZotConfigPath() string { + return AppConfig.zotConfigPath +} + +func GetInput() string { + return AppConfig.input +} + +func SetZotURL(url string) { + AppConfig.zot_url = url +} + +func GetZotURL() string { + return AppConfig.zot_url +} + +func SetRegistry(registry string) { + AppConfig.registry = registry +} + +func GetRegistry() string { + return AppConfig.registry +} + +func SetRepository(repository string) { + AppConfig.repository = repository +} + +func GetRepository() string { + return AppConfig.repository +} + +func SetUserInput(user_input string) { + AppConfig.user_input = user_input +} + +func GetUserInput() string { + return AppConfig.user_input +} + +func SetScheme(scheme string) { + AppConfig.scheme = scheme +} + +func GetScheme() string { + return AppConfig.scheme +} + +func SetAPIVersion(api_version string) { + AppConfig.api_version = api_version +} + +func GetAPIVersion() string { + return AppConfig.api_version +} + +func SetImage(image string) { + AppConfig.image = image +} + +func GetImage() string { + return AppConfig.image } func LoadConfig() (*Config, error) { @@ -24,11 +113,20 @@ func LoadConfig() (*Config, error) { } return &Config{ - LogLevel: viper.GetString("log_level"), - OwnRegistry: viper.GetBool("bring_own_registry"), - OwnRegistryAdr: viper.GetString("own_registry_adr"), - OwnRegistryPort: viper.GetString("own_registry_port"), - ZotConfigPath: viper.GetString("zotConfigPath"), - Input: viper.GetString("url_or_file"), + logLevel: viper.GetString("log_level"), + ownRegistry: viper.GetBool("bring_own_registry"), + ownRegistryAdr: viper.GetString("own_registry_adr"), + ownRegistryPort: viper.GetString("own_registry_port"), + zotConfigPath: viper.GetString("zotConfigPath"), + input: viper.GetString("url_or_file"), }, nil } + +func InitConfig() error { + var err error + AppConfig, err = LoadConfig() + if err != nil { + return err + } + return nil +} diff --git a/internal/images/get-images.go b/internal/images/get-images.go index bac6cdc..e20e38b 100644 --- a/internal/images/get-images.go +++ b/internal/images/get-images.go @@ -10,6 +10,17 @@ import ( "time" ) +type ImageList struct { + RegistryURL string `json:"registryUrl"` + Repositories []struct { + Repository string `json:"repository"` + Images []struct { + Name string `json:"name"` + } `json:"images"` + } `json:"repositories"` +} + + type Image struct { ID int `json:"ID"` Registry string `json:"Registry"` diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..1b647d6 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,131 @@ +package utils + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/images" + "container-registry.com/harbor-satellite/registry" +) + +// / ValidateRegistryAddress validates the registry address and port and returns the URL +func ValidateRegistryAddress(registryAdr, registryPort string) (string, error) { + ip := net.ParseIP(registryAdr) + if ip == nil { + return "", errors.New("invalid IP address") + } + if ip.To4() != nil { + } else { + return "", errors.New("IP address is IPv6 format and unsupported") + } + + return fmt.Sprintf("%s:%s", registryAdr, registryPort), nil +} + +// / HandleOwnRegistry handles the own registry address and port and sets the Zot URL +func HandleOwnRegistry() error { + zotURL, err := ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) + if err != nil { + return err + } + config.SetZotURL(zotURL) + return nil +} + +// LaunchDefaultZotRegistry launches the default Zot registry using the Zot config path +func LaunchDefaultZotRegistry() error { + launch, err := registry.LaunchRegistry(config.GetZotConfigPath()) + if !launch { + return fmt.Errorf("error launching registry: %w", err) + } + if err != nil { + return fmt.Errorf("error launching registry: %w", err) + } + return nil +} + +// Helper function to determine if input is a valid URL +func IsValidURL(input string) bool { + parsedURL, err := url.Parse(input) + return err == nil && parsedURL.Scheme != "" +} + +// GetAbsFilePath gets the absolute file path of the input file path and checks if it exists +func GetAbsFilePath(input string) error { + dir, err := os.Getwd() + if err != nil { + return err + } + absPath := filepath.Join(dir, input) + if _, err := os.Stat(absPath); os.IsNotExist(err) { + return err + } + return nil +} + +// Check if path contains invalid characters +func HasInvalidPathChars(input string) bool { + return strings.ContainsAny(input, "\\:*?\"<>|") +} + +// ParseImagesJsonFile parses the images.json file and decodes it into the ImageList struct +func ParseImagesJsonFile(absPath string, imagesList *images.ImageList) error { + file, err := os.Open(absPath) + if err != nil { + return err + } + defer file.Close() + + if err := json.NewDecoder(file).Decode(imagesList); err != nil { + return err + } + return nil +} + +// Set registry environment variables +func SetRegistryEnvVars(imageList images.ImageList) error { + registryURL := imageList.RegistryURL + registryParts := strings.Split(registryURL, "/") + if len(registryParts) < 3 { + return fmt.Errorf("invalid registryUrl format in images.json") + } + + os.Setenv("REGISTRY", registryParts[2]) + config.SetRegistry(registryParts[2]) + + if len(imageList.Repositories) > 0 { + os.Setenv("REPOSITORY", imageList.Repositories[0].Repository) + config.SetRepository(imageList.Repositories[0].Repository) + } else { + return fmt.Errorf("no repositories found in images.json") + } + + return nil +} + +// SetUrlConfig sets the URL configuration for the input URL and sets the environment variables +func SetUrlConfig(input string) { + os.Setenv("USER_INPUT", input) + config.SetUserInput(input) + parts := strings.SplitN(input, "://", 2) + scheme := parts[0] + "://" + os.Setenv("SCHEME", scheme) + config.SetScheme(scheme) + registryAndPath := parts[1] + registryParts := strings.Split(registryAndPath, "/") + os.Setenv("REGISTRY", registryParts[0]) + config.SetRegistry(registryParts[0]) + os.Setenv("API_VERSION", registryParts[1]) + config.SetAPIVersion(registryParts[1]) + os.Setenv("REPOSITORY", registryParts[2]) + config.SetRepository(registryParts[2]) + os.Setenv("IMAGE", registryParts[3]) + config.SetImage(registryParts[3]) +} diff --git a/main.go b/main.go index e5cb0aa..a477a34 100644 --- a/main.go +++ b/main.go @@ -2,41 +2,26 @@ package main import ( "context" - "encoding/json" - "errors" "fmt" - "net" - "net/url" "os" "os/signal" - "path/filepath" - "strings" "syscall" "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/images" "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/server" "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" - "container-registry.com/harbor-satellite/registry" "golang.org/x/sync/errgroup" - "github.com/spf13/viper" - "github.com/joho/godotenv" + "github.com/rs/zerolog" + "github.com/spf13/viper" ) -type ImageList struct { - RegistryURL string `json:"registryUrl"` - Repositories []struct { - Repository string `json:"repository"` - Images []struct { - Name string `json:"name"` - } `json:"images"` - } `json:"repositories"` -} - func main() { if err := run(); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -45,158 +30,138 @@ func main() { } func run() error { - config, err := config.LoadConfig() - if err != nil { + // Initialize Config and Logger + if err := initConfig(); err != nil { return err } - var fetcher store.ImageFetcher - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM) + ctx, cancel := setupContext() defer cancel() - g, ctx := errgroup.WithContext(ctx) - log := logger.SetupLogger(ctx, config.LogLevel) + g, ctx := errgroup.WithContext(ctx) + log := logger.SetupLogger(ctx, config.GetLogLevel()) log.Info().Msg("Satellite starting") + // Set up router and app + app := setupServerApp(ctx, log) + app.SetupRoutes() + app.SetupServer(g) + + // Handle registry setup + if err := handleRegistrySetup(g, log, cancel); err != nil { + return err + } + + // Process Input (file or URL) + fetcher, err := processInput(ctx, log) + if err != nil { + return err + } + + // Load environment and start satellite + if err := godotenv.Load(); err != nil { + log.Error().Err(err).Msg("Error loading .env file") + return err + } + + ctx, storer := store.NewInMemoryStore(ctx, fetcher) + replicator := replicate.NewReplicator(ctx) + satelliteService := satellite.NewSatellite(ctx, storer, replicator) + + g.Go(func() error { + return satelliteService.Run(ctx) + }) + + log.Info().Msg("Satellite running") + return g.Wait() +} + +func initConfig() error { + if err := config.InitConfig(); err != nil { + return fmt.Errorf("error initializing config: %w", err) + } + return nil +} + +func setupContext() (context.Context, context.CancelFunc) { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + return ctx, cancel +} + +func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { router := server.NewDefaultRouter("/api/v1") router.Use(server.LoggingMiddleware) - app := server.NewApp( + return server.NewApp( router, ctx, log, - config, + config.AppConfig, &server.MetricsRegistrar{}, &server.DebugRegistrar{}, &satellite.SatelliteRegistrar{}, ) +} - app.SetupRoutes() - app.SetupServer(g) - bringOwnRegistry := viper.GetBool("bring_own_registry") - if bringOwnRegistry { - registryAdr := viper.GetString("own_registry_adr") - - // Validate registryAdr format - ip := net.ParseIP(registryAdr) - if ip == nil { - log.Error().Msg("Invalid IP address") - return errors.New("invalid IP address") - } - if ip.To4() != nil { - log.Info().Msg("IP address is valid IPv4") - } else { - log.Error().Msg("IP address is IPv6 format and unsupported") - return errors.New("IP address is IPv6 format and unsupported") +func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context.CancelFunc) error { + if config.GetOwnRegistry() { + if err := utils.HandleOwnRegistry(); err != nil { + log.Error().Err(err).Msg("Error handling own registry") + return err } - registryPort := viper.GetString("own_registry_port") - os.Setenv("ZOT_URL", registryAdr+":"+registryPort) } else { log.Info().Msg("Launching default registry") g.Go(func() error { - launch, err := registry.LaunchRegistry(viper.GetString("zotConfigPath")) - if launch { - cancel() - return err - } - if err != nil { + if err := utils.LaunchDefaultZotRegistry(); err != nil { + log.Error().Err(err).Msg("Error launching default registry") cancel() - log.Error().Err(err).Msg("Failed to launch default registry") return err } + cancel() return nil }) } + return nil +} +func processInput(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { input := viper.GetString("url_or_file") - parsedURL, err := url.Parse(input) - if err != nil || parsedURL.Scheme == "" { - if strings.ContainsAny(input, "\\:*?\"<>|") { - log.Error().Msg("Path contains invalid characters. Please check the configuration.") - return err + if !utils.IsValidURL(config.GetInput()) { + log.Info().Msg("Input is not a valid URL, checking if it is a file path") + if err := validateFilePath(config.GetInput(), log); err != nil { + return nil, err } - dir, err := os.Getwd() - if err != nil { - log.Error().Err(err).Msg("Error getting current directory") - return err - } - absPath := filepath.Join(dir, input) - if _, err := os.Stat(absPath); os.IsNotExist(err) { - log.Error().Err(err).Msg("No URL or file found. Please check the configuration.") - return err - } - log.Info().Msg("Input is a valid file path.") - fetcher = store.FileImageListFetcher(ctx, input) - os.Setenv("USER_INPUT", input) - - // Parse images.json and set environment variables - file, err := os.Open(absPath) - if err != nil { - log.Error().Err(err).Msg("Error opening images.json file") - return err - } - defer file.Close() + return setupFileFetcher(ctx, log) + } - var imageList ImageList - if err := json.NewDecoder(file).Decode(&imageList); err != nil { - log.Error().Err(err).Msg("Error decoding images.json file") - return err - } + log.Info().Msg("Input is a valid URL") + fetcher := store.RemoteImageListFetcher(ctx, input) + utils.SetUrlConfig(input) + return fetcher, nil +} - registryURL := imageList.RegistryURL - registryParts := strings.Split(registryURL, "/") - if len(registryParts) < 3 { - log.Error().Msg("Invalid registryUrl format in images.json") - return errors.New("invalid registryUrl format in images.json") - } - registry := registryParts[2] - os.Setenv("REGISTRY", registry) - - if len(imageList.Repositories) > 0 { - repository := imageList.Repositories[0].Repository - os.Setenv("REPOSITORY", repository) - } else { - log.Error().Msg("No repositories found in images.json") - return errors.New("no repositories found in images.json") - } - } else { - log.Info().Msg("Input is a valid URL.") - fetcher = store.RemoteImageListFetcher(ctx, input) - os.Setenv("USER_INPUT", input) - parts := strings.SplitN(input, "://", 2) - scheme := parts[0] + "://" - os.Setenv("SCHEME", scheme) - registryAndPath := parts[1] - registryParts := strings.Split(registryAndPath, "/") - registry := registryParts[0] - os.Setenv("REGISTRY", registry) - apiVersion := registryParts[1] - os.Setenv("API_VERSION", apiVersion) - repository := registryParts[2] - os.Setenv("REPOSITORY", repository) - image := registryParts[3] - os.Setenv("IMAGE", image) +func validateFilePath(path string, log *zerolog.Logger) error { + if utils.HasInvalidPathChars(path) { + log.Error().Msg("Path contains invalid characters") + return fmt.Errorf("invalid file path: %s", path) } - - err = godotenv.Load() - if err != nil { - log.Error().Err(err).Msg("Error loading.env file") - return err + if err := utils.GetAbsFilePath(path); err != nil { + log.Error().Err(err).Msg("No file found") + return fmt.Errorf("no file found: %s", path) } + return nil +} - ctx, storer := store.NewInMemoryStore(ctx, fetcher) - replicator := replicate.NewReplicator(ctx) - s := satellite.NewSatellite(ctx, storer, replicator) - - g.Go(func() error { - return s.Run(ctx) - }) - log.Info().Msg("Satellite running") - - err = g.Wait() - if err != nil { - log.Error().Err(err).Msg("Error running satellite") - return err +func setupFileFetcher(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { + fetcher := store.FileImageListFetcher(ctx, config.GetInput()) + var imagesList images.ImageList + if err := utils.ParseImagesJsonFile(config.GetInput(), &imagesList); err != nil { + log.Error().Err(err).Msg("Error parsing images.json file") + return nil, err } - return nil + if err := utils.SetRegistryEnvVars(imagesList); err != nil { + log.Error().Err(err).Msg("Error setting registry environment variables") + return nil, err + } + return fetcher, nil } From 4a34c2b1865c8bc4724cb9dc972209ec21458914 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sat, 14 Sep 2024 03:27:00 +0530 Subject: [PATCH 06/10] fixing the logger contex error and coderabbit fixes --- config.toml | 2 +- internal/config/config.go | 50 +++++++++++++++++------------------ internal/server/middleware.go | 3 ++- internal/utils/utils.go | 8 ++++++ logger/logger.go | 6 ----- main.go | 8 +++--- 6 files changed, 40 insertions(+), 37 deletions(-) diff --git a/config.toml b/config.toml index d9ab5a1..8e02820 100644 --- a/config.toml +++ b/config.toml @@ -16,4 +16,4 @@ log_level = "info" # For testing purposes : # https://demo.goharbor.io/v2/myproject/album-server -# /image-list/images.json +# ./image-list/images.json diff --git a/internal/config/config.go b/internal/config/config.go index 8948647..8de8598 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,39 +9,39 @@ import ( var AppConfig *Config type Config struct { - logLevel string - ownRegistry bool - ownRegistryAdr string - ownRegistryPort string - zotConfigPath string - input string - zot_url string - registry string - repository string - user_input string - scheme string - api_version string - image string + log_level string + own_registry bool + own_registry_adr string + own_registry_port string + zot_config_path string + input string + zot_url string + registry string + repository string + user_input string + scheme string + api_version string + image string } func GetLogLevel() string { - return AppConfig.logLevel + return AppConfig.log_level } func GetOwnRegistry() bool { - return AppConfig.ownRegistry + return AppConfig.own_registry } func GetOwnRegistryAdr() string { - return AppConfig.ownRegistryAdr + return AppConfig.own_registry_adr } func GetOwnRegistryPort() string { - return AppConfig.ownRegistryPort + return AppConfig.own_registry_port } func GetZotConfigPath() string { - return AppConfig.zotConfigPath + return AppConfig.zot_config_path } func GetInput() string { @@ -109,16 +109,16 @@ func LoadConfig() (*Config, error) { viper.SetConfigType("toml") viper.AddConfigPath(".") if err := viper.ReadInConfig(); err != nil { - return nil, fmt.Errorf("error reading config file: %w", err) + return nil, fmt.Errorf("error reading config file at path '%s': %w", viper.ConfigFileUsed(), err) } return &Config{ - logLevel: viper.GetString("log_level"), - ownRegistry: viper.GetBool("bring_own_registry"), - ownRegistryAdr: viper.GetString("own_registry_adr"), - ownRegistryPort: viper.GetString("own_registry_port"), - zotConfigPath: viper.GetString("zotConfigPath"), - input: viper.GetString("url_or_file"), + log_level: viper.GetString("log_level"), + own_registry: viper.GetBool("bring_own_registry"), + own_registry_adr: viper.GetString("own_registry_adr"), + own_registry_port: viper.GetString("own_registry_port"), + zot_config_path: viper.GetString("zotConfigPath"), + input: viper.GetString("url_or_file"), }, nil } diff --git a/internal/server/middleware.go b/internal/server/middleware.go index d320124..deae619 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -18,10 +18,11 @@ func LoggingMiddleware(next http.Handler) http.Handler { // Log the request log.Printf( - "%s %s %s %s", + "%s %s %s %s %s", r.RemoteAddr, r.Method, r.URL.Path, + r.Response.Status, time.Since(start), ) }) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 1b647d6..bac9c69 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "container-registry.com/harbor-satellite/internal/config" @@ -25,6 +26,10 @@ func ValidateRegistryAddress(registryAdr, registryPort string) (string, error) { } else { return "", errors.New("IP address is IPv6 format and unsupported") } + port, err := strconv.Atoi(registryPort) + if err != nil || port < 1 || port > 65535 { + return "", errors.New("invalid port number") + } return fmt.Sprintf("%s:%s", registryAdr, registryPort), nil } @@ -91,6 +96,9 @@ func ParseImagesJsonFile(absPath string, imagesList *images.ImageList) error { // Set registry environment variables func SetRegistryEnvVars(imageList images.ImageList) error { + if !IsValidURL(imageList.RegistryURL) { + return fmt.Errorf("invalid registry url format in images.json") + } registryURL := imageList.RegistryURL registryParts := strings.Split(registryURL, "/") if len(registryParts) < 3 { diff --git a/logger/logger.go b/logger/logger.go index 13c85af..78664dc 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -48,9 +48,3 @@ func FromContext(ctx context.Context) *zerolog.Logger { } return logger } - -func SetupLogger(ctx context.Context, logLevel string) *zerolog.Logger{ - ctx = AddLoggerToContext(ctx, logLevel) - logger := FromContext(ctx) - return logger -} diff --git a/main.go b/main.go index a477a34..d0fc417 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,6 @@ import ( "github.com/joho/godotenv" "github.com/rs/zerolog" - "github.com/spf13/viper" ) func main() { @@ -39,7 +38,8 @@ func run() error { defer cancel() g, ctx := errgroup.WithContext(ctx) - log := logger.SetupLogger(ctx, config.GetLogLevel()) + ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) + log := logger.FromContext(ctx) log.Info().Msg("Satellite starting") // Set up router and app @@ -125,8 +125,8 @@ func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context. } func processInput(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { - input := viper.GetString("url_or_file") - if !utils.IsValidURL(config.GetInput()) { + input := config.GetInput() + if !utils.IsValidURL(input) { log.Info().Msg("Input is not a valid URL, checking if it is a file path") if err := validateFilePath(config.GetInput(), log); err != nil { return nil, err From 5f75ed8b38bd9dc4ba8f05daf96c1e752deb4663 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 15 Sep 2024 15:50:48 +0530 Subject: [PATCH 07/10] making the secure push compulsory until specified --- .env | 4 ++- internal/config/config.go | 46 +++++++++++++++++++++++++++++++++ internal/replicate/replicate.go | 28 ++++++++++++++++---- internal/store/http-fetch.go | 26 ++++++++++++++----- main.go | 7 ----- 5 files changed, 91 insertions(+), 20 deletions(-) diff --git a/.env b/.env index f6a4ec5..6a08828 100644 --- a/.env +++ b/.env @@ -1,4 +1,6 @@ HARBOR_USERNAME=admin HARBOR_PASSWORD=Harbor12345 ZOT_URL="127.0.0.1:8585" -TOKEN="" \ No newline at end of file +TOKEN="" +ENV=dev +USE_UNSECURE=true diff --git a/internal/config/config.go b/internal/config/config.go index 8de8598..067554a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,7 +2,9 @@ package config import ( "fmt" + "os" + "github.com/joho/godotenv" "github.com/spf13/viper" ) @@ -22,6 +24,10 @@ type Config struct { scheme string api_version string image string + harbor_password string + harbor_username string + env string + use_unsecure bool } func GetLogLevel() string { @@ -104,6 +110,18 @@ func GetImage() string { return AppConfig.image } +func UseUnsecure() bool { + return AppConfig.use_unsecure +} + +func GetHarborPassword() string { + return AppConfig.harbor_password +} + +func GetHarborUsername() string { + return AppConfig.harbor_username +} + func LoadConfig() (*Config, error) { viper.SetConfigName("config") viper.SetConfigType("toml") @@ -112,6 +130,17 @@ func LoadConfig() (*Config, error) { return nil, fmt.Errorf("error reading config file at path '%s': %w", viper.ConfigFileUsed(), err) } + // Load environment and start satellite + if err := godotenv.Load(); err != nil { + return &Config{}, fmt.Errorf("error loading .env file: %w", err) + } + var use_unsecure bool + if os.Getenv("USE_UNSECURE") == "true" { + use_unsecure = true + } else { + use_unsecure = false + } + return &Config{ log_level: viper.GetString("log_level"), own_registry: viper.GetBool("bring_own_registry"), @@ -119,6 +148,11 @@ func LoadConfig() (*Config, error) { own_registry_port: viper.GetString("own_registry_port"), zot_config_path: viper.GetString("zotConfigPath"), input: viper.GetString("url_or_file"), + harbor_password: os.Getenv("HARBOR_PASSWORD"), + harbor_username: os.Getenv("HARBOR_USERNAME"), + env: os.Getenv("ENV"), + zot_url: os.Getenv("ZOT_URL"), + use_unsecure: use_unsecure, }, nil } @@ -128,5 +162,17 @@ func InitConfig() error { if err != nil { return err } + /// Print all the configuration + fmt.Println("Configuration:") + fmt.Println("Log Level: ", AppConfig.log_level) + fmt.Println("Own Registry: ", AppConfig.own_registry) + fmt.Println("Own Registry Address: ", AppConfig.own_registry_adr) + fmt.Println("Own Registry Port: ", AppConfig.own_registry_port) + fmt.Println("Zot Config Path: ", AppConfig.zot_config_path) + fmt.Println("Input: ", AppConfig.input) + fmt.Println("Harbor Password: ", AppConfig.harbor_password) + fmt.Println("Harbor Username: ", AppConfig.harbor_username) + fmt.Println("Environment: ", AppConfig.env) + fmt.Println("Use Unsecure: ", AppConfig.use_unsecure) return nil } diff --git a/internal/replicate/replicate.go b/internal/replicate/replicate.go index 5a2267a..59bb805 100644 --- a/internal/replicate/replicate.go +++ b/internal/replicate/replicate.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/store" "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" @@ -20,7 +21,12 @@ type Replicator interface { DeleteExtraImages(ctx context.Context, imgs []store.Image) error } -type BasicReplicator struct{} +type BasicReplicator struct { + username string + password string + use_unsecure bool + zot_url string +} type ImageInfo struct { Name string `json:"name"` @@ -37,7 +43,12 @@ type RegistryInfo struct { } func NewReplicator(context context.Context) Replicator { - return &BasicReplicator{} + return &BasicReplicator{ + username: config.GetHarborUsername(), + password: config.GetHarborPassword(), + use_unsecure: config.UseUnsecure(), + zot_url: config.GetZotURL(), + } } func (r *BasicReplicator) Replicate(ctx context.Context, image string) error { @@ -177,9 +188,12 @@ func CopyImage(ctx context.Context, imageName string) error { Username: username, Password: password, }) - + options := []crane.Option{crane.WithAuth(auth)} + if config.UseUnsecure() { + options = append(options, crane.Insecure) + } // Pull the image with authentication - srcImage, err := crane.Pull(imageName, crane.WithAuth(auth), crane.Insecure) + srcImage, err := crane.Pull(imageName, options...) if err != nil { log.Error().Msgf("Failed to pull image: %v", err) return fmt.Errorf("failed to pull image: %w", err) @@ -188,7 +202,11 @@ func CopyImage(ctx context.Context, imageName string) error { } // Push the image to the destination registry - err = crane.Push(srcImage, destRef, crane.Insecure) + push_options := []crane.Option{} + if config.UseUnsecure() { + push_options = append(push_options, crane.Insecure) + } + err = crane.Push(srcImage, destRef, push_options...) if err != nil { log.Error().Msgf("Failed to push image: %v", err) return fmt.Errorf("failed to push image: %w", err) diff --git a/internal/store/http-fetch.go b/internal/store/http-fetch.go index 922a454..4dc7b7c 100644 --- a/internal/store/http-fetch.go +++ b/internal/store/http-fetch.go @@ -11,13 +11,18 @@ import ( "strings" "time" + "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" ) type RemoteImageList struct { - BaseURL string + BaseURL string + username string + password string + use_unsecure bool + zot_url string } type TagListResponse struct { @@ -27,7 +32,11 @@ type TagListResponse struct { func RemoteImageListFetcher(ctx context.Context, url string) *RemoteImageList { return &RemoteImageList{ - BaseURL: url, + BaseURL: url, + username: config.GetHarborUsername(), + password: config.GetHarborPassword(), + use_unsecure: config.UseUnsecure(), + zot_url: config.GetZotURL(), } } @@ -106,15 +115,18 @@ func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (strin // Encode credentials for Basic Authentication username := os.Getenv("HARBOR_USERNAME") password := os.Getenv("HARBOR_PASSWORD") + auth := &authn.Basic{Username: username, Password: password} + // Prepare options for crane.Digest + options := []crane.Option{crane.WithAuth(auth)} + if client.use_unsecure { + options = append(options, crane.Insecure) + } // Use crane.Digest to get the digest of the image - digest, err := crane.Digest(imageRef, crane.WithAuth(&authn.Basic{ - Username: username, - Password: password, - }), crane.Insecure) + digest, err := crane.Digest(imageRef, options...) if err != nil { log.Error().Msgf("failed to get digest using crane: %v", err) - return "", nil + return "", err } return digest, nil diff --git a/main.go b/main.go index d0fc417..68d8d1f 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,6 @@ import ( "container-registry.com/harbor-satellite/logger" "golang.org/x/sync/errgroup" - "github.com/joho/godotenv" "github.com/rs/zerolog" ) @@ -58,12 +57,6 @@ func run() error { return err } - // Load environment and start satellite - if err := godotenv.Load(); err != nil { - log.Error().Err(err).Msg("Error loading .env file") - return err - } - ctx, storer := store.NewInMemoryStore(ctx, fetcher) replicator := replicate.NewReplicator(ctx) satelliteService := satellite.NewSatellite(ctx, storer, replicator) From 8ce1b197edad4600dddd6a4b0e22116c757c48e9 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 15 Sep 2024 16:30:51 +0530 Subject: [PATCH 08/10] coderabbit review --- internal/config/config.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 067554a..93ae830 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -162,17 +162,5 @@ func InitConfig() error { if err != nil { return err } - /// Print all the configuration - fmt.Println("Configuration:") - fmt.Println("Log Level: ", AppConfig.log_level) - fmt.Println("Own Registry: ", AppConfig.own_registry) - fmt.Println("Own Registry Address: ", AppConfig.own_registry_adr) - fmt.Println("Own Registry Port: ", AppConfig.own_registry_port) - fmt.Println("Zot Config Path: ", AppConfig.zot_config_path) - fmt.Println("Input: ", AppConfig.input) - fmt.Println("Harbor Password: ", AppConfig.harbor_password) - fmt.Println("Harbor Username: ", AppConfig.harbor_username) - fmt.Println("Environment: ", AppConfig.env) - fmt.Println("Use Unsecure: ", AppConfig.use_unsecure) return nil } From c29de050c77d65051bce5019d0208e7e9672a537 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Wed, 18 Sep 2024 23:25:55 +0530 Subject: [PATCH 09/10] fixing the test cases --- test/e2e/satellite_test.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test/e2e/satellite_test.go b/test/e2e/satellite_test.go index 9d7435f..fefab89 100644 --- a/test/e2e/satellite_test.go +++ b/test/e2e/satellite_test.go @@ -44,14 +44,10 @@ func setupSourceRegistry( client *dagger.Client, ctx context.Context, ) (*dagger.Service, error) { - // socket to connect to host Docker - socket := client.Host().UnixSocket("/var/run/docker.sock") container, err := client.Container(). From("registry:2"). WithExposedPort(5000). - WithUnixSocket("/var/run/docker.sock", socket). - WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). WithEnvVariable("CACHEBUSTER", time.Now().String()). AsService().Start(ctx) @@ -66,14 +62,10 @@ func setupDestinationRegistry( client *dagger.Client, ctx context.Context, ) (*dagger.Service, error) { - // socket to connect to host Docker - socket := client.Host().UnixSocket("/var/run/docker.sock") container, err := client.Container(). From("registry:2"). WithExposedPort(5000). - WithUnixSocket("/var/run/docker.sock", socket). - WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). WithEnvVariable("CACHEBUSTER", time.Now().String()). AsService().Start(ctx) @@ -89,13 +81,9 @@ func pushImageToSourceRegistry( client *dagger.Client, source *dagger.Service, ) { - // socket to connect to host Docker - socket := client.Host().UnixSocket("/var/run/docker.sock") container := client.Container(). From("docker:dind"). - WithUnixSocket("/var/run/docker.sock", socket). - WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). WithEnvVariable("CACHEBUSTER", time.Now().String()). WithServiceBinding("source", source) @@ -120,7 +108,6 @@ func buildSatellite( source *dagger.Service, dest *dagger.Service, ) { - socket := client.Host().UnixSocket("/var/run/docker.sock") var PATH_TO_CONFIG string if ABS { PATH_TO_CONFIG = absolute_path @@ -142,8 +129,6 @@ func buildSatellite( WithWorkdir(appDir). WithServiceBinding("source", source). WithServiceBinding("dest", dest). - WithUnixSocket("/var/run/docker.sock", socket). - WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). WithEnvVariable("CACHEBUSTER", time.Now().String()). WithExec([]string{"cat", "config.toml"}). WithFile("./config.toml", configFile). @@ -152,7 +137,7 @@ func buildSatellite( WithExposedPort(9090). WithExec([]string{"./" + appBinary}). AsService() - // Get the response + satellite_container := client.Container(). From("golang:alpine"). WithServiceBinding("satellite", container). From 67e49c286aa95f4a07232e58bcc0dd9977ec67dd Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Thu, 19 Sep 2024 00:50:21 +0530 Subject: [PATCH 10/10] adding running tests in github workflow --- .github/workflows/build.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 69b437b..ca4da6e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,42 @@ on: workflow_dispatch: jobs: + run-tests: + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '>=1.22' + + - name: Install Curl + run: sudo apt-get install curl + + - name: Install Docker + run: | + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + + - name: Install Dagger CLI + run: | + cd /usr/local + curl -L https://dl.dagger.io/dagger/install.sh | sudo sh + cd - + + - name: Generated the dagger go code + run: dagger develop + + - name: Run Test Cases + run: dagger run go test ./... -v -count=1 + dagger-build-satellite: + needs: run-tests runs-on: ubuntu-latest permissions: contents: write @@ -43,6 +78,7 @@ jobs: # cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} build-ground-control: + needs: run-tests runs-on: ubuntu-latest permissions: contents: write