diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 4e52721..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index ae8aed1..424ddb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.env # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # diff --git a/config.toml b/config.toml index 0c25118..93b9ebd 100644 --- a/config.toml +++ b/config.toml @@ -1,12 +1,13 @@ # Wether to us the built-in Zot registry or not -bring_own_registry = false +bring_own_registry = true # URL of own registry -own_registry_adr = "127.0.0.1:8585" +own_registry_adr = "127.0.0.1:5000" # URL of remote registry OR local file path -url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" +# url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" + url_or_file = "http://localhost:5001/v2/library/busybox" # For testing purposes : # https://demo.goharbor.io/v2/myproject/album-server -# /image-list/images.json \ No newline at end of file +# /image-list/images.json diff --git a/internal/replicate/replicate.go b/internal/replicate/replicate.go index 7b0b3e3..aa73454 100644 --- a/internal/replicate/replicate.go +++ b/internal/replicate/replicate.go @@ -40,7 +40,6 @@ func NewReplicator() Replicator { } func (r *BasicReplicator) Replicate(ctx context.Context, image string) error { - source := getPullSource(image) if source != "" { @@ -98,7 +97,8 @@ func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Im func getPullSource(image string) string { input := os.Getenv("USER_INPUT") - if os.Getenv("SCHEME") == "https://" { + scheme := os.Getenv("SCHEME") + if strings.HasPrefix(scheme, "http://") || strings.HasPrefix(scheme, "https://") { url := os.Getenv("HOST") + "/" + os.Getenv("REGISTRY") + "/" + image return url } else { @@ -115,7 +115,6 @@ func getPullSource(image string) string { return registryURL + repositoryName + "/" + image } - } func getFileInfo(input string) (*RegistryInfo, error) { @@ -150,8 +149,10 @@ func CopyImage(imageName string) error { return fmt.Errorf("ZOT_URL environment variable is not set") } - srcRef := imageName - destRef := zotUrl + "/" + imageName + // Clean up the image name by removing any host part + cleanedImageName := removeHostName(imageName) + destRef := fmt.Sprintf("%s/%s", zotUrl, cleanedImageName) + fmt.Println("Destination reference:", destRef) // Get credentials from environment variables username := os.Getenv("HARBOR_USERNAME") @@ -166,7 +167,7 @@ func CopyImage(imageName string) error { }) // Pull the image with authentication - srcImage, err := crane.Pull(srcRef, crane.WithAuth(auth)) + srcImage, err := crane.Pull(imageName, crane.WithAuth(auth), crane.Insecure) if err != nil { fmt.Printf("Failed to pull image: %v\n", err) return fmt.Errorf("failed to pull image: %w", err) @@ -176,7 +177,7 @@ func CopyImage(imageName string) error { } // Push the image to the destination registry - err = crane.Push(srcImage, destRef) + err = crane.Push(srcImage, destRef, crane.Insecure) if err != nil { fmt.Printf("Failed to push image: %v\n", err) return fmt.Errorf("failed to push image: %w", err) @@ -195,3 +196,13 @@ func CopyImage(imageName string) error { return nil } + +// take only the parts after the hostname +func removeHostName(imageName string) string { + parts := strings.Split(imageName, "/") + if len(parts) > 1 { + return strings.Join(parts[1:], "/") + } + + return imageName +} diff --git a/internal/store/http-fetch.go b/internal/store/http-fetch.go index f878710..309287a 100644 --- a/internal/store/http-fetch.go +++ b/internal/store/http-fetch.go @@ -99,7 +99,7 @@ func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (strin digest, err := crane.Digest(imageRef, crane.WithAuth(&authn.Basic{ Username: username, Password: password, - })) + }), crane.Insecure) if err != nil { fmt.Printf("failed to fetch digest for %s: %v\n", imageRef, err) return "", nil diff --git a/internal/store/in-memory-store.go b/internal/store/in-memory-store.go index 6652e5e..de3a702 100644 --- a/internal/store/in-memory-store.go +++ b/internal/store/in-memory-store.go @@ -155,7 +155,6 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { fmt.Println("No changes detected in the store") return nil, nil } - } func (s *inMemoryStore) Add(ctx context.Context, digest string, image string) error { @@ -201,7 +200,6 @@ func (s *inMemoryStore) RemoveImage(ctx context.Context, image string) error { // TODO: Rework complicated logic and add support for multiple repositories // checkImageAndDigest checks if the image exists in the store and if the digest matches the image reference func (s *inMemoryStore) checkImageAndDigest(digest string, image string) bool { - // Check if the received image exists in the store for storeDigest, storeImage := range s.images { if storeImage == image { @@ -236,19 +234,18 @@ func (s *inMemoryStore) checkImageAndDigest(digest string, image string) bool { // If adding was successful, return true, else return false err := s.Add(context.Background(), digest, image) return err != nil - } func GetLocalDigest(ctx context.Context, tag string) (string, error) { - zotUrl := os.Getenv("ZOT_URL") userURL := os.Getenv("USER_INPUT") // Remove extra characters from the URLs userURL = userURL[strings.Index(userURL, "//")+2:] userURL = strings.ReplaceAll(userURL, "/v2", "") + regUrl := removeHostName(userURL) // Construct the URL for fetching the digest - url := zotUrl + "/" + userURL + ":" + tag + url := zotUrl + "/" + regUrl + ":" + tag // Use crane.Digest to get the digest of the image digest, err := crane.Digest(url) @@ -258,3 +255,13 @@ func GetLocalDigest(ctx context.Context, tag string) (string, error) { return digest, nil } + +// Split the imageName by "/" and take only the parts after the hostname +func removeHostName(imageName string) string { + parts := strings.Split(imageName, "/") + if len(parts) > 1 { + return strings.Join(parts[1:], "/") + } + + return imageName +} diff --git a/main.go b/main.go index df3c04b..0201807 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,6 @@ import ( "os" "os/signal" "path/filepath" - "regexp" "strings" "syscall" "time" @@ -85,13 +84,13 @@ func run() error { registryAdr := viper.GetString("own_registry_adr") // Validate registryAdr format - matched, err := regexp.MatchString(`^127\.0\.0\.1:\d{1,5}$`, registryAdr) - if err != nil { - return fmt.Errorf("error validating registry address: %w", err) - } - if !matched { - return fmt.Errorf("invalid registry address format: %s", registryAdr) - } + // matched, err := regexp.MatchString(`^127\.0\.0\.1:\d{1,5}$`, registryAdr) + // if err != nil { + // return fmt.Errorf("error validating registry address: %w", err) + // } + // if matched { + // return fmt.Errorf("invalid registry address format: %s", registryAdr) + // } os.Setenv("ZOT_URL", registryAdr) fmt.Println("Registry URL set to:", registryAdr) } else { @@ -105,7 +104,6 @@ func run() error { cancel() return err } - }) } diff --git a/registry/.DS_Store b/registry/.DS_Store deleted file mode 100644 index d6e99f1..0000000 Binary files a/registry/.DS_Store and /dev/null differ diff --git a/registry/config.json b/registry/config.json index 01970c0..34e2b82 100644 --- a/registry/config.json +++ b/registry/config.json @@ -10,4 +10,4 @@ "log": { "level": "" } -} \ No newline at end of file +} diff --git a/test/e2e/satellite_test.go b/test/e2e/satellite_test.go new file mode 100644 index 0000000..3175d31 --- /dev/null +++ b/test/e2e/satellite_test.go @@ -0,0 +1,178 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "dagger.io/dagger" + "github.com/stretchr/testify/assert" +) + +const ( + appDir = "/app" + appBinary = "app" + sourceFile = "main.go" +) + +func TestSatellite(t *testing.T) { + ctx := context.Background() + + // Initialize Dagger client + client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr)) + assert.NoError(t, err, "Failed to connect to Dagger") + defer client.Close() + + // Set up Source Registry + source, err := setupSourceRegistry(t, client, ctx) + assert.NoError(t, err, "Failed to set up source registry") + + // Set up Destination registry + dest, err := setupDestinationRegistry(t, client, ctx) + assert.NoError(t, err, "Failed to set up destination registry") + + // Push images to Source registry + pushImageToSourceRegistry(t, ctx, client, source) + assert.NoError(t, err, "Failed to upload image to source registry") + + // Build & Run Satellite + buildSatellite(t, client, ctx, source, dest) + assert.NoError(t, err, "Failed to build and run Satellite") +} + +// Setup Source Registry as a Dagger Service +func setupSourceRegistry( + t *testing.T, + client *dagger.Client, + ctx context.Context, +) (*dagger.Service, error) { + // socket to connect to host Docker + socket := client.Host().UnixSocket("/var/run/docker.sock") + + container, err := client.Container(). + From("registry:2"). + WithExposedPort(5000). + WithUnixSocket("/var/run/docker.sock", socket). + WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). + WithEnvVariable("CACHEBUSTER", time.Now().String()). + AsService().Start(ctx) + + assert.NoError(t, err, "Failed setting up source registry.") + + return container, nil +} + +// Setup Destination Registry as a Dagger Service +func setupDestinationRegistry( + t *testing.T, + client *dagger.Client, + ctx context.Context, +) (*dagger.Service, error) { + // socket to connect to host Docker + socket := client.Host().UnixSocket("/var/run/docker.sock") + + container, err := client.Container(). + From("registry:2"). + WithExposedPort(5000). + WithUnixSocket("/var/run/docker.sock", socket). + WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). + WithEnvVariable("CACHEBUSTER", time.Now().String()). + AsService().Start(ctx) + + assert.NoError(t, err, "Failed setting up destination registry") + + return container, nil +} + +// Push image to the Source registry +func pushImageToSourceRegistry( + t *testing.T, + ctx context.Context, + client *dagger.Client, + source *dagger.Service, +) { + // socket to connect to host Docker + socket := client.Host().UnixSocket("/var/run/docker.sock") + + container := client.Container(). + From("docker:dind"). + WithUnixSocket("/var/run/docker.sock", socket). + WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). + WithEnvVariable("CACHEBUSTER", time.Now().String()). + WithServiceBinding("source", source) + + // add crane & push images + container = container.WithExec([]string{"apk", "add", "crane"}). + WithExec([]string{"docker", "pull", "busybox:1.36"}). + WithExec([]string{"docker", "pull", "busybox:stable"}). + WithExec([]string{"crane", "copy", "busybox:1.36", "source:5000/library/busybox:1.36", "--insecure"}). + WithExec([]string{"crane", "copy", "busybox:stable", "source:5000/library/busybox:stable", "--insecure"}). + WithExec([]string{"crane", "digest", "source:5000/library/busybox:1.36", "--insecure"}). + WithExec([]string{"crane", "digest", "source:5000/library/busybox:stable", "--insecure"}) + + // check pushed images exist + container = container.WithExec([]string{"crane", "catalog", "source:5000", "--insecure"}) + + stdOut, err := container.Stdout(ctx) + assert.NoError(t, err, "Failed to print stdOut in pushing Image to Source") + + fmt.Println(stdOut) +} + +// buildSatellite and test test the connection +func buildSatellite( + t *testing.T, + client *dagger.Client, + ctx context.Context, + source *dagger.Service, + dest *dagger.Service, +) { + socket := client.Host().UnixSocket("/var/run/docker.sock") + + // Get the directory + parentDir, err := getProjectDir() + assert.NoError(t, err, "Failed to get Project Directory") + + // Use the directory path in Dagger + dir := client.Host().Directory(parentDir) + + // Get configuration file on the host + configFile := client.Host().File("./testdata/config.toml") + + // Configure and build the Satellite + container := client.Container().From("golang:alpine").WithDirectory(appDir, dir). + WithWorkdir(appDir). + WithServiceBinding("source", source). + WithServiceBinding("dest", dest). + WithUnixSocket("/var/run/docker.sock", socket). + WithEnvVariable("DOCKER_HOST", "unix:///var/run/docker.sock"). + WithEnvVariable("CACHEBUSTER", time.Now().String()). + WithExec([]string{"cat", "config.toml"}). + WithFile("./config.toml", configFile). + WithExec([]string{"cat", "config.toml"}). + WithExec([]string{"apk", "add", "crane"}). + WithExec([]string{"crane", "-v", "catalog", "source:5000", "--insecure"}). + WithExec([]string{"crane", "digest", "source:5000/library/busybox:stable", "--insecure"}). + WithExec([]string{"go", "build", "-o", appBinary, sourceFile}). + WithExposedPort(9090). + WithExec([]string{"go", "run", "./test/e2e/test.go"}) + + assert.NoError(t, err, "Test failed in buildSatellite") + + stdOut, err := container.Stdout(ctx) + assert.NoError(t, err, "Failed to get stdOut in Satellite") + + fmt.Println(stdOut) +} + +// Gets the directory of the project +func getProjectDir() (string, error) { + currentDir, err := os.Getwd() + if err != nil { + return "", err + } + return filepath.Abs(filepath.Join(currentDir, "../..")) +} diff --git a/test/e2e/test.go b/test/e2e/test.go new file mode 100644 index 0000000..53ad8f7 --- /dev/null +++ b/test/e2e/test.go @@ -0,0 +1,53 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "os" + "os/exec" + "strings" +) + +func main() { + // Command to execute + cmd := exec.Command("go", "run", "./main.go") + + // Get stdout pipe + stdout, err := cmd.StdoutPipe() + if err != nil { + log.Fatalf("Error creating stdout pipe: %v", err) + } + + // Start the command + if err := cmd.Start(); err != nil { + log.Fatalf("Error starting command: %v", err) + } + + // Create a scanner to read the command output line by line + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + fmt.Println(line) // Print each line of output + + lineParts := strings.Split(line, "--") + // Check if the line contains "----" + if len(lineParts) > 2 { + fmt.Println("Satellite is Working...\nExiting...") + if err := cmd.Process.Kill(); err != nil { + fmt.Println("Error killing process:", err) + } + os.Exit(0) // Exit the program + } + } + + // Handle any scanner error + if err := scanner.Err(); err != nil { + log.Fatalf("Error reading stdout: %v", err) + } + + // Wait for the command to finish + if err := cmd.Wait(); err != nil { + log.Fatalf("Command execution failed: %v", err) + } +} diff --git a/test/e2e/testdata/config.toml b/test/e2e/testdata/config.toml new file mode 100644 index 0000000..06d9736 --- /dev/null +++ b/test/e2e/testdata/config.toml @@ -0,0 +1,7 @@ +bring_own_registry = true +own_registry_adr = "dest:5000" +url_or_file = "http://source:5000/v2/library/busybox" + +# Additional test cases need to be handled. +# url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" +# url_or_file = "http://localhost:5001/v2/library/busybox"