add list all repositories in registry support

Signed-off-by: Adphi <philippe.adrien.nousse@gmail.com>

Dockerfile

@@ -24,11 +24,13 @@ COPY cmd ./cmd
 COPY pkg ./pkg
 
 RUN go build -trimpath -ldflags="-s -w" -o artifact-registry ./cmd/artifact-registry
+RUN go build -trimpath -ldflags="-s -w" -o lkar ./cmd/lkar
 
 FROM alpine:latest
 
 RUN apk --no-cache add ca-certificates
 
 COPY --from=0 /app/artifact-registry /usr/local/bin/artifact-registry
+COPY --from=0 /app/lkar /usr/local/bin/lkar
 
 ENTRYPOINT ["/usr/local/bin/artifact-registry"]

Makefile

@@ -0,0 +1,17 @@
+IMAGE := "linkacloud/artifact-registry"
+MODULE = go.linka.cloud/artifact-registry
+
+install: docker-build
+	@go generate ./...
+	@go install -trimpath -ldflags "-s -w -X '$(MODULE).Version=$(VERSION)' -X '$(MODULE).BuildDate=$(shell date)'" ./cmd/artifact-registry
+	@go install -trimpath -ldflags "-s -w -X '$(MODULE).Version=$(VERSION)' -X '$(MODULE).BuildDate=$(shell date)'" ./cmd/lkar
+
+docker: docker-build docker-push
+
+.PHONY: docker-build
+docker-build:
+	@docker build -t $(IMAGE) .
+
+.PHONY: docker-push
+docker-push:
+	@docker push $(IMAGE)

cmd/artifact-registry/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	artifact_registry "go.linka.cloud/artifact-registry"
 	"go.linka.cloud/artifact-registry/pkg/logger"
 	"go.linka.cloud/artifact-registry/pkg/packages"
 	"go.linka.cloud/artifact-registry/pkg/repository"
@@ -93,9 +94,17 @@ var (
 			}
 		},
 	}
+	cmdVersion = &cobra.Command{
+		Use: "version",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Println(artifact_registry.Version)
+			fmt.Println(artifact_registry.BuildDate)
+		},
+	}
 )
 
 func main() {
+	cmd.AddCommand(cmdVersion)
 	cmd.Flags().StringVar(&addr, "addr", envDefault(EnvAddr, addr), "address to listen on [$"+EnvAddr+"]")
 	cmd.Flags().StringVar(&backend, "backend", envDefault(EnvBackend, backend), "registry backend [$"+EnvBackend+"]")
 	cmd.Flags().StringVar(&aesKey, "aes-key", envDefault(EnvKey, aesKey), "AES key to encrypt the repositories keys [$"+EnvKey+"]")

cmd/lkar/main.go

@@ -15,7 +15,6 @@
 package main
 
 import (
-	"errors"
 	"os"
 	"strings"
 
@@ -54,10 +53,9 @@ func setup(cmd *cobra.Command, args []string) error {
 	arg := args[0]
 	parts := strings.Split(arg, "/")
 	registry = parts[0]
-	if len(parts) == 1 {
-		return errors.New("missing repository")
+	if len(parts) > 1 {
+		repository = strings.Join(parts[1:], "/")
 	}
-	repository = strings.Join(parts[1:], "/")
 	var err error
 	if format, err = printer.ParseFormat(output); err != nil {
 		return err
@@ -76,6 +74,9 @@ func setup(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	if repository == "" {
+		return nil
+	}
 	if creds.Username == "" && creds.Password == "" {
 		creds, err = credsStore.Get(cmd.Context(), registry)
 		if err != nil {

cmd/lkar/packages.go

@@ -24,6 +24,15 @@ func newPkgCmd(typ string) *cobra.Command {
 	pkgCmd := &cobra.Command{
 		Use:   typ,
 		Short: fmt.Sprintf("Root command for %s management", typ),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := setup(cmd, args); err != nil {
+				return err
+			}
+			if repository == "" {
+				return fmt.Errorf("repository part is required")
+			}
+			return nil
+		},
 	}
 	pkgCmd.AddCommand(
 		newPkgListCmd(typ),

cmd/lkar/repo.go

@@ -36,7 +36,11 @@ var (
 		Aliases: []string{"repo", "repos"},
 		Args:    cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			req, err := http.NewRequestWithContext(cmd.Context(), http.MethodGet, url()+"/_repositories/"+repository, nil)
+			u := url() + "/_repositories/" + repository
+			if repository == "" {
+				u = url() + "/_repositories"
+			}
+			req, err := http.NewRequestWithContext(cmd.Context(), http.MethodGet, u, nil)
 			if err != nil {
 				return err
 			}

cmd/lkar/utils.go

@@ -45,6 +45,9 @@ func url(typ ...string) string {
 }
 
 func repoURL() string {
+	if repository == "" {
+		return registry
+	}
 	return registry + "/" + repository
 }
 

cmd/lkar/version.go

@@ -0,0 +1,37 @@
+// Copyright 2023 Linka Cloud  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	artifact_registry "go.linka.cloud/artifact-registry"
+)
+
+var (
+	cmdVersion = &cobra.Command{
+		Use: "version",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Println(artifact_registry.Version)
+			fmt.Println(artifact_registry.BuildDate)
+		},
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(cmdVersion)
+}

pkg/repository/repository.go

@@ -17,15 +17,14 @@ package repository
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"golang.org/x/sync/errgroup"
-	"oras.land/oras-go/v2/errdef"
 	"oras.land/oras-go/v2/registry/remote"
 
 	cache2 "go.linka.cloud/artifact-registry/pkg/cache"
@@ -74,7 +73,7 @@ func Auth(w http.ResponseWriter, r *http.Request) {
 	}
 	for _, v := range ts {
 		if _, err := repo.Manifests().Resolve(ctx, v); err != nil {
-			if errors.Is(err, errdef.ErrNotFound) {
+			if storage.IsNotFound(err) {
 				continue
 			}
 			if err != nil {
@@ -87,41 +86,34 @@ func Auth(w http.ResponseWriter, r *http.Request) {
 	http.Error(w, "No repository found", http.StatusNotFound)
 }
 
-func ListRepositories(w http.ResponseWriter, r *http.Request) {
-	ctx := auth.Context(r.Context(), r)
-	o := storage.Options(ctx)
-	name, typ := mux.Vars(r)["repo"], mux.Vars(r)["type"]
-	reg, err := remote.NewRegistry(o.Host())
-	if err != nil {
-		storage.Error(w, err)
-		return
-	}
-	reg.Client = o.Client(ctx, o.Host())
+func listImageRepositories(ctx context.Context, reg *remote.Registry, name string, typ ...string) ([]*Repository, error) {
 	repo, err := reg.Repository(ctx, name)
 	if err != nil {
-		storage.Error(w, err)
-		return
+		if storage.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, err
 	}
+	typ = slices.Filter(typ, func(s string) bool {
+		return s != ""
+	})
 	var tags []string
-	if typ == "" {
-		if err := repo.Tags(ctx, "", func(s []string) error {
-			tags = append(tags, s...)
-			return nil
-		}); err != nil {
-			storage.Error(w, err)
-			return
-		}
-		tags = slices.Filter(tags, func(s string) bool {
-			return s == "apk" || s == "deb" || s == "rpm"
-		})
+	if len(typ) == 0 {
+		tags = packages.Providers()
 	} else {
-		tags = []string{typ}
+		tags = typ
 	}
-	out := make([]*Repository, len(tags))
+	var (
+		out []*Repository
+		mu  sync.Mutex
+	)
 	g, ctx := errgroup.WithContext(ctx)
 	fn := func(i int, typ string) error {
 		desc, err := repo.Manifests().Resolve(ctx, typ)
 		if err != nil {
+			if storage.IsNotFound(err) {
+				return nil
+			}
 			return err
 		}
 		var m ocispec.Manifest
@@ -144,7 +136,7 @@ func ListRepositories(w http.ResponseWriter, r *http.Request) {
 			return err
 		}
 		r := &Repository{
-			Name:        o.Host() + "/" + name,
+			Name:        storage.Options(ctx).Host() + "/" + name,
 			Type:        typ,
 			LastUpdated: &t,
 		}
@@ -158,7 +150,9 @@ func ListRepositories(w http.ResponseWriter, r *http.Request) {
 				r.Metadata.Count++
 			}
 		}
-		out[i] = r
+		mu.Lock()
+		out = append(out, r)
+		mu.Unlock()
 		return nil
 	}
 	for i, v := range tags {
@@ -170,6 +164,46 @@ func ListRepositories(w http.ResponseWriter, r *http.Request) {
 			return nil
 		})
 	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func ListRepositories(w http.ResponseWriter, r *http.Request) {
+	ctx := auth.Context(r.Context(), r)
+	o := storage.Options(ctx)
+	typ := mux.Vars(r)["type"]
+	reg, err := remote.NewRegistry(o.Host())
+	if err != nil {
+		storage.Error(w, err)
+		return
+	}
+	reg.Client = o.Client(ctx, o.Host())
+	var repos []string
+	if err := reg.Repositories(ctx, "", func(r []string) error {
+		repos = append(repos, r...)
+		return nil
+	}); err != nil {
+		storage.Error(w, err)
+		return
+	}
+	var out []*Repository
+	var mu sync.Mutex
+	g, ctx := errgroup.WithContext(ctx)
+	for _, v := range repos {
+		v := v
+		g.Go(func() error {
+			rs, err := listImageRepositories(ctx, reg, v, typ)
+			if err != nil {
+				return fmt.Errorf("%s: %w", v, err)
+			}
+			mu.Lock()
+			out = append(out, rs...)
+			mu.Unlock()
+			return nil
+		})
+	}
 	if err := g.Wait(); err != nil {
 		storage.Error(w, err)
 		return
@@ -180,6 +214,23 @@ func ListRepositories(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func ListImageRepositories(w http.ResponseWriter, r *http.Request) {
+	ctx := auth.Context(r.Context(), r)
+	o := storage.Options(ctx)
+	name, typ := mux.Vars(r)["repo"], mux.Vars(r)["type"]
+	reg, err := remote.NewRegistry(o.Host())
+	if err != nil {
+		storage.Error(w, err)
+		return
+	}
+	reg.Client = o.Client(ctx, o.Host())
+	out, err := listImageRepositories(ctx, reg, name, typ)
+	if err := json.NewEncoder(w).Encode(out); err != nil {
+		storage.Error(w, err)
+		return
+	}
+}
+
 func Packages(w http.ResponseWriter, r *http.Request) {
 	ctx := auth.Context(r.Context(), r)
 	typ, repo := mux.Vars(r)["type"], mux.Vars(r)["repo"]
@@ -206,7 +257,8 @@ func Packages(w http.ResponseWriter, r *http.Request) {
 
 func Init(_ context.Context, r *mux.Router, domain string) error {
 	r.Path("/_auth/{repo:.+}").Methods(http.MethodGet, http.MethodPost).HandlerFunc(Auth)
-	r.Path("/_repositories/{repo:.+}").Methods(http.MethodGet).HandlerFunc(ListRepositories)
+	r.Path("/_repositories").Methods(http.MethodGet).HandlerFunc(ListRepositories)
+	r.Path("/_repositories/{repo:.+}").Methods(http.MethodGet).HandlerFunc(ListImageRepositories)
 	subs := []*mux.Router{r.PathPrefix("/{type}/_packages/").Subrouter()}
 	if domain != "" {
 		subs = append(subs, r.Host("{type}."+domain+"/_packages").Subrouter())

pkg/storage/errors.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 	"os"
 
+	"oras.land/oras-go/v2/errdef"
 	"oras.land/oras-go/v2/registry/remote/errcode"
 )
 
@@ -37,6 +38,13 @@ func ErrCode(err error) int {
 	return http.StatusInternalServerError
 }
 
+func IsNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, os.ErrNotExist) || ErrCode(err) == http.StatusNotFound || errors.Is(err, errdef.ErrNotFound)
+}
+
 func Error(w http.ResponseWriter, err error) {
 	var ec *errcode.ErrorResponse
 	switch {

version.go

@@ -0,0 +1,20 @@
+// Copyright 2023 Linka Cloud  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifact_registry
+
+var (
+	Version   = "dev"
+	BuildDate = ""
+)