From 6fa38f2cf2dabb69d3727d6161b06d33dbdb9a87 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 24 Mar 2023 15:37:45 +0000 Subject: [PATCH 1/8] Add Alpine package registry. --- custom/conf/app.example.ini | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + models/packages/alpine/search.go | 53 +++ models/packages/descriptor.go | 3 + models/packages/package.go | 7 + models/packages/package_file.go | 2 +- models/packages/package_property.go | 38 ++ models/user/setting.go | 5 + modules/packages/alpine/metadata.go | 236 ++++++++++++ modules/packages/alpine/metadata_test.go | 143 +++++++ modules/setting/packages.go | 2 + options/locale/locale_en-US.ini | 9 + public/img/svg/gitea-alpine.svg | 1 + routers/api/packages/alpine/alpine.go | 233 +++++++++++ routers/api/packages/api.go | 14 + routers/api/packages/nuget/nuget.go | 6 +- routers/api/v1/packages/package.go | 2 +- routers/web/user/package.go | 35 +- services/forms/package_form.go | 2 +- services/packages/alpine/repository.go | 361 ++++++++++++++++++ services/packages/packages.go | 84 +++- templates/package/content/alpine.tmpl | 52 +++ templates/package/metadata/alpine.tmpl | 5 + templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + tests/integration/api_packages_alpine_test.go | 229 +++++++++++ web_src/svg/gitea-alpine.svg | 2 + 27 files changed, 1508 insertions(+), 22 deletions(-) create mode 100644 models/packages/alpine/search.go create mode 100644 modules/packages/alpine/metadata.go create mode 100644 modules/packages/alpine/metadata_test.go create mode 100644 public/img/svg/gitea-alpine.svg create mode 100644 routers/api/packages/alpine/alpine.go create mode 100644 services/packages/alpine/repository.go create mode 100644 templates/package/content/alpine.tmpl create mode 100644 templates/package/metadata/alpine.tmpl create mode 100644 tests/integration/api_packages_alpine_test.go create mode 100644 web_src/svg/gitea-alpine.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index e53ed7ad9fd8b..c30cd2c3bb0f6 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2488,6 +2488,8 @@ ROUTER = console ;LIMIT_TOTAL_OWNER_COUNT = -1 ;; Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_TOTAL_OWNER_SIZE = -1 +;; Maximum size of an Alpine upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_ALPINE = -1 ;; Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_CARGO = -1 ;; Maximum size of a Chef upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 4b9c519cd80df..03743bf874a1b 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1240,6 +1240,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` - `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maximum count of package versions a single owner can have (`-1` means no limits) - `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_ALPINE`: **-1**: Maximum size of an Alpine upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CARGO`: **-1**: Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CHEF`: **-1**: Maximum size of a Chef upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_COMPOSER`: **-1**: Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/models/packages/alpine/search.go b/models/packages/alpine/search.go new file mode 100644 index 0000000000000..77eccb90ed5ef --- /dev/null +++ b/models/packages/alpine/search.go @@ -0,0 +1,53 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "context" + + packages_model "code.gitea.io/gitea/models/packages" + alpine_module "code.gitea.io/gitea/modules/packages/alpine" +) + +// GetBranches gets all available branches +func GetBranches(ctx context.Context, ownerID int64) ([]string, error) { + return packages_model.GetDistinctPropertyValues( + ctx, + packages_model.TypeAlpine, + ownerID, + packages_model.PropertyTypeFile, + alpine_module.PropertyBranch, + nil, + ) +} + +// GetRepositories gets all available repositories for the given branch +func GetRepositories(ctx context.Context, ownerID int64, branch string) ([]string, error) { + return packages_model.GetDistinctPropertyValues( + ctx, + packages_model.TypeAlpine, + ownerID, + packages_model.PropertyTypeFile, + alpine_module.PropertyRepository, + &packages_model.DistinctPropertyDependency{ + Name: alpine_module.PropertyBranch, + Value: branch, + }, + ) +} + +// GetArchitectures gets all available architectures for the given repository +func GetArchitectures(ctx context.Context, ownerID int64, repository string) ([]string, error) { + return packages_model.GetDistinctPropertyValues( + ctx, + packages_model.TypeAlpine, + ownerID, + packages_model.PropertyTypeFile, + alpine_module.PropertyArchitecture, + &packages_model.DistinctPropertyDependency{ + Name: alpine_module.PropertyRepository, + Value: repository, + }, + ) +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 06699b5d572b2..cfb069f18262e 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/alpine" "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/packages/composer" @@ -132,6 +133,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc var metadata interface{} switch p.Type { + case TypeAlpine: + metadata = &alpine.VersionMetadata{} case TypeCargo: metadata = &cargo.Metadata{} case TypeChef: diff --git a/models/packages/package.go b/models/packages/package.go index ccc9257c31235..19d07861c11b6 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -30,6 +30,7 @@ type Type string // List of supported packages const ( + TypeAlpine Type = "alpine" TypeCargo Type = "cargo" TypeChef Type = "chef" TypeComposer Type = "composer" @@ -49,6 +50,7 @@ const ( ) var TypeList = []Type{ + TypeAlpine, TypeCargo, TypeChef, TypeComposer, @@ -70,6 +72,8 @@ var TypeList = []Type{ // Name gets the name of the package type func (pt Type) Name() string { switch pt { + case TypeAlpine: + return "Alpine" case TypeCargo: return "Cargo" case TypeChef: @@ -109,6 +113,8 @@ func (pt Type) Name() string { // SVGName gets the name of the package type svg image func (pt Type) SVGName() string { switch pt { + case TypeAlpine: + return "gitea-alpine" case TypeCargo: return "gitea-cargo" case TypeChef: @@ -154,6 +160,7 @@ type Package struct { Name string `xorm:"NOT NULL"` LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` + IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` } // TryInsertPackage inserts a package. If a package exists already, ErrDuplicatePackage is returned diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 97e7a0d4070a9..6647f3fcdc715 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -118,7 +118,7 @@ func DeleteFileByID(ctx context.Context, fileID int64) error { // PackageFileSearchOptions are options for SearchXXX methods type PackageFileSearchOptions struct { OwnerID int64 - PackageType string + PackageType Type VersionID int64 Query string CompositeKey string diff --git a/models/packages/package_property.go b/models/packages/package_property.go index e03b12c9df4dd..e0170016cfc9c 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -7,6 +7,8 @@ import ( "context" "code.gitea.io/gitea/models/db" + + "xorm.io/builder" ) func init() { @@ -81,3 +83,39 @@ func DeletePropertyByName(ctx context.Context, refType PropertyType, refID int64 _, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Delete(&PackageProperty{}) return err } + +type DistinctPropertyDependency struct { + Name string + Value string +} + +// GetDistinctPropertyValues returns all distinct property values for a given type. +// Optional: Search only in dependence of another property. +func GetDistinctPropertyValues(ctx context.Context, packageType Type, ownerID int64, refType PropertyType, propertyName string, dep *DistinctPropertyDependency) ([]string, error) { + var cond builder.Cond = builder.Eq{ + "package_property.ref_type": refType, + "package_property.name": propertyName, + "package.type": packageType, + "package.owner_id": ownerID, + } + if dep != nil { + innerCond := builder. + Expr("pp.ref_id = package_property.ref_id"). + And(builder.Eq{ + "pp.ref_type": refType, + "pp.name": dep.Name, + "pp.value": dep.Value, + }) + cond = cond.And(builder.Exists(builder.Select("pp.ref_id").From("package_property pp").Where(innerCond))) + } + + values := make([]string, 0, 5) + return values, db.GetEngine(ctx). + Table("package_property"). + Distinct("package_property.value"). + Join("INNER", "package_file", "package_file.id = package_property.ref_id"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Find(&values) +} diff --git a/models/user/setting.go b/models/user/setting.go index aec79b756bf14..a41e494db9b54 100644 --- a/models/user/setting.go +++ b/models/user/setting.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/cache" setting_module "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) @@ -42,6 +43,10 @@ func (err ErrUserSettingIsNotExist) Error() string { return fmt.Sprintf("Setting[%s] is not exist", err.Key) } +func (err ErrUserSettingIsNotExist) Unwrap() error { + return util.ErrNotExist +} + // IsErrUserSettingIsNotExist return true if err is ErrSettingIsNotExist func IsErrUserSettingIsNotExist(err error) bool { _, ok := err.(ErrUserSettingIsNotExist) diff --git a/modules/packages/alpine/metadata.go b/modules/packages/alpine/metadata.go new file mode 100644 index 0000000000000..c2d0caffa1259 --- /dev/null +++ b/modules/packages/alpine/metadata.go @@ -0,0 +1,236 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "crypto/sha1" + "encoding/base64" + "io" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" +) + +var ( + ErrMissingPKGINFOFile = util.NewInvalidArgumentErrorf("PKGINFO file is missing") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +const ( + PropertyMetadata = "alpine.metadata" + PropertyBranch = "alpine.branch" + PropertyRepository = "alpine.repository" + PropertyArchitecture = "alpine.architecture" + + SettingKeyPrivate = "alpine.key.private" + SettingKeyPublic = "alpine.key.public" + + RepositoryPackage = "_alpine" + RepositoryVersion = "_repository" +) + +// https://wiki.alpinelinux.org/wiki/Apk_spec + +// Package represents an Alpine package +type Package struct { + Name string + Version string + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Metadata of an Alpine package +type VersionMetadata struct { + Description string `json:"description,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Maintainer string `json:"maintainer,omitempty"` +} + +type FileMetadata struct { + Checksum string `json:"checksum"` + Packager string `json:"packager,omitempty"` + BuildDate int64 `json:"build_date,omitempty"` + Size int64 `json:"size,omitempty"` + Architecture string `json:"architecture,omitempty"` + Origin string `json:"origin,omitempty"` + CommitHash string `json:"commit_hash,omitempty"` + InstallIf string `json:"install_if,omitempty"` + Provides []string `json:"provides,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` +} + +// ParsePackage parses the Alpine package file +func ParsePackage(r io.Reader) (*Package, error) { + // Alpine packages are concated .tar.gz streams. Usually the first stream contains the package metadata. + + br := bufio.NewReader(r) // needed for gzip Multistream + + h := sha1.New() + + gzr, err := gzip.NewReader(&teeByteReader{br, h}) + if err != nil { + return nil, err + } + defer gzr.Close() + + for { + gzr.Multistream(false) + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Name == ".PKGINFO" { + p, err := ParsePackageInfo(tr) + if err != nil { + return nil, err + } + + // drain the reader + for { + if _, err := tr.Next(); err != nil { + break + } + } + + p.FileMetadata.Checksum = "Q1" + base64.StdEncoding.EncodeToString(h.Sum(nil)) + + return p, nil + } + } + + h = sha1.New() + + err = gzr.Reset(&teeByteReader{br, h}) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + + return nil, ErrMissingPKGINFOFile +} + +// ParsePackageInfo parses a PKGINFO file to retrieve the metadata of an Alpine package +func ParsePackageInfo(r io.Reader) (*Package, error) { + p := &Package{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { + continue + } + + i := strings.IndexRune(line, '=') + if i == -1 { + continue + } + + key := strings.TrimSpace(line[:i]) + value := strings.TrimSpace(line[i+1:]) + + switch key { + case "pkgname": + p.Name = value + case "pkgver": + p.Version = value + case "pkgdesc": + p.VersionMetadata.Description = value + case "url": + p.VersionMetadata.ProjectURL = value + case "builddate": + n, err := strconv.ParseInt(value, 10, 64) + if err == nil { + p.FileMetadata.BuildDate = n + } + case "size": + n, err := strconv.ParseInt(value, 10, 64) + if err == nil { + p.FileMetadata.Size = n + } + case "arch": + p.FileMetadata.Architecture = value + case "origin": + p.FileMetadata.Origin = value + case "commit": + p.FileMetadata.CommitHash = value + case "maintainer": + p.VersionMetadata.Maintainer = value + case "packager": + p.FileMetadata.Packager = value + case "license": + p.VersionMetadata.License = value + case "install_if": + p.FileMetadata.InstallIf = value + case "provides": + if value != "" { + p.FileMetadata.Provides = append(p.FileMetadata.Provides, value) + } + case "depend": + if value != "" { + p.FileMetadata.Dependencies = append(p.FileMetadata.Dependencies, value) + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + + if p.Name == "" { + return nil, ErrInvalidName + } + + if p.Version == "" { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + p.VersionMetadata.ProjectURL = "" + } + + return p, nil +} + +// Same as io.TeeReader but implements io.ByteReader +type teeByteReader struct { + r *bufio.Reader + w io.Writer +} + +func (t *teeByteReader) Read(p []byte) (int, error) { + n, err := t.r.Read(p) + if n > 0 { + if n, err := t.w.Write(p[:n]); err != nil { + return n, err + } + } + return n, err +} + +func (t *teeByteReader) ReadByte() (byte, error) { + b, err := t.r.ReadByte() + if err == nil { + if _, err := t.w.Write([]byte{b}); err != nil { + return 0, err + } + } + return b, err +} diff --git a/modules/packages/alpine/metadata_test.go b/modules/packages/alpine/metadata_test.go new file mode 100644 index 0000000000000..2a3c48ffb9a27 --- /dev/null +++ b/modules/packages/alpine/metadata_test.go @@ -0,0 +1,143 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + packageDescription = "Package Description" + packageProjectURL = "https://gitea.io" + packageMaintainer = "KN4CK3R " +) + +func createPKGINFOContent(name, version string) []byte { + return []byte(`pkgname = ` + name + ` +pkgver = ` + version + ` +pkgdesc = ` + packageDescription + ` +url = ` + packageProjectURL + ` +# comment +builddate = 1678834800 +packager = Gitea +size = 123456 +arch = aarch64 +origin = origin +commit = 1111e709613fbc979651b09ac2bc27c6591a9999 +maintainer = ` + packageMaintainer + ` +license = MIT +depend = common +install_if = value +depend = gitea +provides = common +provides = gitea`) +} + +func TestParsePackage(t *testing.T) { + createPackage := func(name string, content []byte) io.Reader { + names := []string{"first.stream", name} + contents := [][]byte{{0}, content} + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + + for i := range names { + if i != 0 { + zw.Close() + zw.Reset(&buf) + } + + tw := tar.NewWriter(zw) + hdr := &tar.Header{ + Name: names[i], + Mode: 0o600, + Size: int64(len(contents[i])), + } + tw.WriteHeader(hdr) + tw.Write(contents[i]) + tw.Close() + } + + zw.Close() + + return &buf + } + + t.Run("MissingPKGINFOFile", func(t *testing.T) { + data := createPackage("dummy.txt", []byte{}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + assert.ErrorIs(t, err, ErrMissingPKGINFOFile) + }) + + t.Run("InvalidPKGINFOFile", func(t *testing.T) { + data := createPackage(".PKGINFO", []byte{}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + assert.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("Valid", func(t *testing.T) { + data := createPackage(".PKGINFO", createPKGINFOContent(packageName, packageVersion)) + + p, err := ParsePackage(data) + assert.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, "Q1SRYURM5+uQDqfHSwTnNIOIuuDVQ=", p.FileMetadata.Checksum) + }) +} + +func TestParsePackageInfo(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + data := createPKGINFOContent("", packageVersion) + + p, err := ParsePackageInfo(bytes.NewReader(data)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("InvalidVersion", func(t *testing.T) { + data := createPKGINFOContent(packageName, "") + + p, err := ParsePackageInfo(bytes.NewReader(data)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + data := createPKGINFOContent(packageName, packageVersion) + + p, err := ParsePackageInfo(bytes.NewReader(data)) + assert.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, packageDescription, p.VersionMetadata.Description) + assert.Equal(t, packageMaintainer, p.VersionMetadata.Maintainer) + assert.Equal(t, packageProjectURL, p.VersionMetadata.ProjectURL) + assert.Equal(t, "MIT", p.VersionMetadata.License) + assert.Empty(t, p.FileMetadata.Checksum) + assert.Equal(t, "Gitea ", p.FileMetadata.Packager) + assert.EqualValues(t, 1678834800, p.FileMetadata.BuildDate) + assert.EqualValues(t, 123456, p.FileMetadata.Size) + assert.Equal(t, "aarch64", p.FileMetadata.Architecture) + assert.Equal(t, "origin", p.FileMetadata.Origin) + assert.Equal(t, "1111e709613fbc979651b09ac2bc27c6591a9999", p.FileMetadata.CommitHash) + assert.Equal(t, "value", p.FileMetadata.InstallIf) + assert.ElementsMatch(t, []string{"common", "gitea"}, p.FileMetadata.Provides) + assert.ElementsMatch(t, []string{"common", "gitea"}, p.FileMetadata.Dependencies) + }) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index ac0ad62bca3d1..866a4b1387435 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -25,6 +25,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 + LimitSizeAlpine int64 LimitSizeCargo int64 LimitSizeChef int64 LimitSizeComposer int64 @@ -68,6 +69,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { } Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") + Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index afcf9ade04da7..49f61d7468be5 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3172,6 +3172,15 @@ versions.on = on versions.view_all = View all dependency.id = ID dependency.version = Version +alpine.registry = Setup this registry by adding the url in your /etc/apk/repositories file: +alpine.registry.key = Add the registry public RSA key into the /etc/apk/keys folder to verify the index signature: +alpine.registry.info = Choose <branch> and <repository> from the list below. +alpine.install = To install the package, run the following command: +alpine.documentation = For more information on the Alpine registry, see the documentation. +alpine.repository = Repository Info +alpine.repository.branches = Branches +alpine.repository.repositories = Repositories +alpine.repository.architectures = Architectures cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: cargo.documentation = For more information on the Cargo registry, see the documentation. diff --git a/public/img/svg/gitea-alpine.svg b/public/img/svg/gitea-alpine.svg new file mode 100644 index 0000000000000..b2abe872690ce --- /dev/null +++ b/public/img/svg/gitea-alpine.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/alpine/alpine.go b/routers/api/packages/alpine/alpine.go new file mode 100644 index 0000000000000..4e186b31f234a --- /dev/null +++ b/routers/api/packages/alpine/alpine.go @@ -0,0 +1,233 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + packages_module "code.gitea.io/gitea/modules/packages" + alpine_module "code.gitea.io/gitea/modules/packages/alpine" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + alpine_service "code.gitea.io/gitea/services/packages/alpine" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +func GetRepositoryKey(ctx *context.Context) { + _, pub, err := alpine_service.GetOrCreateKeyPair(ctx.Package.Owner.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{ + ContentType: "application/x-pem-file", + Filename: fmt.Sprintf("%s@%s.rsa.pub", ctx.Package.Owner.LowerName, setting.Domain), + }) +} + +func GetRepositoryFile(ctx *context.Context) { + pv, err := alpine_service.GetOrCreateRepositoryVersion(ctx.Package.Owner.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + s, pf, err := packages_service.GetFileStreamByPackageVersion( + ctx, + pv, + &packages_service.PackageFileInfo{ + Filename: alpine_service.IndexFilename, + CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")), + }, + ) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +func UploadPackageFile(ctx *context.Context) { + branch := strings.TrimSpace(ctx.Params("branch")) + repository := strings.TrimSpace(ctx.Params("repository")) + if branch == "" || repository == "" { + apiError(ctx, http.StatusBadRequest, "invalid branch or repository") + return + } + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pck, err := alpine_module.ParsePackage(buf) + if err != nil { + if errors.Is(err, util.ErrInvalidArgument) || err == io.EOF { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + fileMetadataRaw, err := json.Marshal(pck.FileMetadata) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeAlpine, + Name: pck.Name, + Version: pck.Version, + }, + Creator: ctx.Doer, + Metadata: pck.VersionMetadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%s-%s.apk", pck.Name, pck.Version), + CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, pck.FileMetadata.Architecture), + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + Properties: map[string]string{ + alpine_module.PropertyBranch: branch, + alpine_module.PropertyRepository: repository, + alpine_module.PropertyArchitecture: pck.FileMetadata.Architecture, + alpine_module.PropertyMetadata: string(fileMetadataRaw), + }, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile: + apiError(ctx, http.StatusBadRequest, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if err := alpine_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, branch, repository, pck.FileMetadata.Architecture); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +func DownloadPackageFile(ctx *context.Context) { + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + PackageType: packages_model.TypeAlpine, + Query: ctx.Params("filename"), + CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + s, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +func DeletePackageFile(ctx *context.Context) { + branch, repository, architecture := ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture") + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + PackageType: packages_model.TypeAlpine, + Query: ctx.Params("filename"), + CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + if err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx.Doer, pfs[0]); err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if err := alpine_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, branch, repository, architecture); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusNoContent) +} diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index c0c7b117f696b..0386285ec3345 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/packages/alpine" "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/chef" "code.gitea.io/gitea/routers/api/packages/composer" @@ -75,6 +76,19 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }) r.Group("/{username}", func() { + r.Group("/alpine", func() { + r.Get("/key", alpine.GetRepositoryKey) + r.Group("/{branch}/{repository}", func() { + r.Put("", reqPackageAccess(perm.AccessModeWrite), alpine.UploadPackageFile) + r.Group("/{architecture}", func() { + r.Get("/APKINDEX.tar.gz", alpine.GetRepositoryFile) + r.Group("/{filename}", func() { + r.Get("", alpine.DownloadPackageFile) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), alpine.DeletePackageFile) + }) + }) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/cargo", func() { r.Group("/api/v1/crates", func() { r.Get("", cargo.SearchPackages) diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 3418bf9959a96..dbfe8aa2fbd63 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -475,7 +475,7 @@ func UploadSymbolPackage(ctx *context.Context) { Version: np.Version, } - _, _, err = packages_service.AddFileToExistingPackage( + _, err = packages_service.AddFileToExistingPackage( pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ @@ -501,7 +501,7 @@ func UploadSymbolPackage(ctx *context.Context) { } for _, pdb := range pdbs { - _, _, err := packages_service.AddFileToExistingPackage( + _, err := packages_service.AddFileToExistingPackage( pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ @@ -585,7 +585,7 @@ func DownloadSymbolFile(ctx *context.Context) { pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ OwnerID: ctx.Package.Owner.ID, - PackageType: string(packages_model.TypeNuGet), + PackageType: packages_model.TypeNuGet, Query: filename, Properties: map[string]string{ nuget_module.PropertySymbolID: strings.ToLower(guid), diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 200dc5aaf1401..1c81a20a468b2 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [cargo, chef, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, swift, vagrant] + // enum: [alpine, cargo, chef, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, swift, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/user/package.go b/routers/web/user/package.go index a9acc5281feea..470a48ca33ab4 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -14,8 +14,10 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + alpine_module "code.gitea.io/gitea/modules/packages/alpine" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" @@ -163,6 +165,33 @@ func ViewPackageVersion(ctx *context.Context) { ctx.Data["IsPackagesPage"] = true ctx.Data["PackageDescriptor"] = pd + switch pd.Package.Type { + case packages_model.TypeContainer: + ctx.Data["RegistryHost"] = setting.Packages.RegistryHost + case packages_model.TypeAlpine: + branches := make(container.Set[string]) + repositories := make(container.Set[string]) + architectures := make(container.Set[string]) + + for _, f := range pd.Files { + for _, pp := range f.Properties { + switch pp.Name { + case alpine_module.PropertyBranch: + branches.Add(pp.Value) + case alpine_module.PropertyRepository: + repositories.Add(pp.Value) + case alpine_module.PropertyArchitecture: + architectures.Add(pp.Value) + } + } + } + + ctx.Data["Branches"] = branches.Values() + ctx.Data["Repositories"] = repositories.Values() + ctx.Data["Architectures"] = architectures.Values() + ctx.Data["Domain"] = setting.Domain + } + var ( total int64 pvs []*packages_model.PackageVersion @@ -170,8 +199,6 @@ func ViewPackageVersion(ctx *context.Context) { ) switch pd.Package.Type { case packages_model.TypeContainer: - ctx.Data["RegistryHost"] = setting.Packages.RegistryHost - pvs, total, err = container_model.SearchImageTags(ctx, &container_model.ImageTagsSearchOptions{ Paginator: db.NewAbsoluteListOptions(0, 5), PackageID: pd.Package.ID, @@ -183,10 +210,6 @@ func ViewPackageVersion(ctx *context.Context) { PackageID: pd.Package.ID, IsInternal: util.OptionalBoolFalse, }) - if err != nil { - ctx.ServerError("SearchVersions", err) - return - } } if err != nil { ctx.ServerError("", err) diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 699d0fe44f967..d1721bea99ce2 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"` + Type string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/alpine/repository.go b/services/packages/alpine/repository.go new file mode 100644 index 0000000000000..b33b7ff937bbe --- /dev/null +++ b/services/packages/alpine/repository.go @@ -0,0 +1,361 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + alpine_model "code.gitea.io/gitea/models/packages/alpine" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + alpine_module "code.gitea.io/gitea/modules/packages/alpine" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + packages_service "code.gitea.io/gitea/services/packages" +) + +const IndexFilename = "APKINDEX.tar.gz" + +// GetOrCreateRepositoryVersion gets or creates the internal repository package +// The Alpine registry needs multiple index files which are stored in this package. +func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) { + var repositoryVersion *packages_model.PackageVersion + + return repositoryVersion, db.WithTx(db.DefaultContext, func(ctx context.Context) error { + p := &packages_model.Package{ + OwnerID: ownerID, + Type: packages_model.TypeAlpine, + Name: alpine_module.RepositoryPackage, + LowerName: alpine_module.RepositoryPackage, + IsInternal: true, + } + var err error + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return err + } + } + + pv := &packages_model.PackageVersion{ + PackageID: p.ID, + CreatorID: ownerID, + Version: alpine_module.RepositoryVersion, + LowerVersion: alpine_module.RepositoryVersion, + IsInternal: true, + MetadataJSON: "null", + } + if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { + if err != packages_model.ErrDuplicatePackageVersion { + log.Error("Error inserting package version: %v", err) + return err + } + } + + repositoryVersion = pv + + return nil + }) +} + +// GetOrCreateKeyPair gets or creates the RSA keys used to sign repository files +func GetOrCreateKeyPair(ownerID int64) (string, string, error) { + priv, err := user_model.GetSetting(ownerID, alpine_module.SettingKeyPrivate) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + + pub, err := user_model.GetSetting(ownerID, alpine_module.SettingKeyPublic) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + + if priv == "" || pub == "" { + priv, pub, err = util.GenerateKeyPair(4096) + if err != nil { + return "", "", err + } + + if err := user_model.SetUserSetting(ownerID, alpine_module.SettingKeyPrivate, priv); err != nil { + return "", "", err + } + + if err := user_model.SetUserSetting(ownerID, alpine_module.SettingKeyPublic, pub); err != nil { + return "", "", err + } + } + + return priv, pub, nil +} + +// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures +func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error { + pv, err := GetOrCreateRepositoryVersion(ownerID) + if err != nil { + return err + } + + // 1. Delete all existing repository files + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return err + } + + for _, pf := range pfs { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + // 2. (Re)Build repository files for existing packages + branches, err := alpine_model.GetBranches(ctx, ownerID) + if err != nil { + return err + } + for _, branch := range branches { + repositories, err := alpine_model.GetRepositories(ctx, ownerID, branch) + if err != nil { + return err + } + for _, repository := range repositories { + architectures, err := alpine_model.GetArchitectures(ctx, ownerID, repository) + if err != nil { + return err + } + for _, architecture := range architectures { + if err := buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture); err != nil { + return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", branch, repository, architecture, err) + } + } + } + } + + return nil +} + +// BuildSpecificRepositoryFiles builds index files for the repository +func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, branch, repository, architecture string) error { + pv, err := GetOrCreateRepositoryVersion(ownerID) + if err != nil { + return err + } + + return buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture) +} + +type packageData struct { + Package *packages_model.Package + Version *packages_model.PackageVersion + Blob *packages_model.PackageBlob + VersionMetadata *alpine_module.VersionMetadata + FileMetadata *alpine_module.FileMetadata +} + +type packageCache = map[*packages_model.PackageFile]*packageData + +// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format +func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error { + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + OwnerID: ownerID, + PackageType: packages_model.TypeAlpine, + Query: "%.apk", + Properties: map[string]string{ + alpine_module.PropertyBranch: branch, + alpine_module.PropertyRepository: repository, + alpine_module.PropertyArchitecture: architecture, + }, + }) + if err != nil { + return err + } + + // Delete the package indices if there are no packages + if len(pfs) == 0 { + pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture)) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return err + } + + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + return packages_model.DeleteFileByID(ctx, pf.ID) + } + + // Cache data needed for all repository files + cache := make(packageCache) + for _, pf := range pfs { + pv, err := packages_model.GetVersionByID(ctx, pf.VersionID) + if err != nil { + return err + } + p, err := packages_model.GetPackageByID(ctx, pv.PackageID) + if err != nil { + return err + } + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) + if err != nil { + return err + } + pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, alpine_module.PropertyMetadata) + if err != nil { + return err + } + + pd := &packageData{ + Package: p, + Version: pv, + Blob: pb, + } + + if err := json.Unmarshal([]byte(pv.MetadataJSON), &pd.VersionMetadata); err != nil { + return err + } + if len(pps) > 0 { + if err := json.Unmarshal([]byte(pps[0].Value), &pd.FileMetadata); err != nil { + return err + } + } + + cache[pf] = pd + } + + var buf bytes.Buffer + for _, pf := range pfs { + pd := cache[pf] + + fmt.Fprintf(&buf, "C:%s\n", pd.FileMetadata.Checksum) + fmt.Fprintf(&buf, "P:%s\n", pd.Package.Name) + fmt.Fprintf(&buf, "V:%s\n", pd.Version.Version) + fmt.Fprintf(&buf, "A:%s\n", pd.FileMetadata.Architecture) + if pd.VersionMetadata.Description != "" { + fmt.Fprintf(&buf, "T:%s\n", pd.VersionMetadata.Description) + } + if pd.VersionMetadata.ProjectURL != "" { + fmt.Fprintf(&buf, "U:%s\n", pd.VersionMetadata.ProjectURL) + } + if pd.VersionMetadata.License != "" { + fmt.Fprintf(&buf, "L:%s\n", pd.VersionMetadata.License) + } + fmt.Fprintf(&buf, "S:%d\n", pd.Blob.Size) + fmt.Fprintf(&buf, "I:%d\n", pd.FileMetadata.Size) + fmt.Fprintf(&buf, "o:%s\n", pd.FileMetadata.Origin) + fmt.Fprintf(&buf, "m:%s\n", pd.VersionMetadata.Maintainer) + fmt.Fprintf(&buf, "t:%d\n", pd.FileMetadata.BuildDate) + if pd.FileMetadata.CommitHash != "" { + fmt.Fprintf(&buf, "c:%s\n", pd.FileMetadata.CommitHash) + } + if len(pd.FileMetadata.Dependencies) > 0 { + fmt.Fprintf(&buf, "D:%s\n", strings.Join(pd.FileMetadata.Dependencies, " ")) + } + if len(pd.FileMetadata.Provides) > 0 { + fmt.Fprintf(&buf, "p:%s\n", strings.Join(pd.FileMetadata.Provides, " ")) + } + fmt.Fprint(&buf, "\n") + } + + unsignedIndexContent, _ := packages_module.NewHashedBuffer(32 * 1024 * 1024) + h := sha1.New() + + if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), "APKINDEX", buf.Bytes(), true); err != nil { + return err + } + + priv, _, err := GetOrCreateKeyPair(ownerID) + if err != nil { + return err + } + + privPem, _ := pem.Decode([]byte(priv)) + if privPem == nil { + return fmt.Errorf("failed to decode private key pem") + } + + privKey, err := x509.ParsePKCS1PrivateKey(privPem.Bytes) + if err != nil { + return err + } + + sign, err := rsa.SignPKCS1v15(rand.Reader, privKey, crypto.SHA1, h.Sum(nil)) + if err != nil { + return err + } + + owner, err := user_model.GetUserByID(ctx, ownerID) + if err != nil { + return err + } + + signedIndexContent, _ := packages_module.NewHashedBuffer(32 * 1024 * 1024) + + if err := writeGzipStream( + signedIndexContent, + fmt.Sprintf(".SIGN.RSA.%s@%s.rsa.pub", owner.LowerName, setting.Domain), + sign, + false, + ); err != nil { + return err + } + + if _, err := io.Copy(signedIndexContent, unsignedIndexContent); err != nil { + return err + } + + _, err = packages_service.AddFileToPackageVersionInternal( + repoVersion, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: IndexFilename, + CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture), + }, + Creator: user_model.NewGhostUser(), + Data: signedIndexContent, + IsLead: false, + OverwriteExisting: true, + }, + ) + return err +} + +func writeGzipStream(w io.Writer, filename string, content []byte, addTarEnd bool) error { + zw := gzip.NewWriter(w) + defer zw.Close() + + tw := tar.NewWriter(zw) + if addTarEnd { + defer tw.Close() + } + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := tw.Write(content); err != nil { + return err + } + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index dd5c63470b8b2..8fdaba2d2738e 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -187,19 +187,34 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } // AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned -func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { +func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) { + return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + if err != nil { + return nil, nil, false, err + } + + return addFileToPackageVersion(ctx, pv, pvi, pfci) + }) +} + +// AddFileToPackageVersionInternal adds a file to the package +// This method skips quota checks and should only be used for system-managed packages. +func AddFileToPackageVersionInternal(pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) { + return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + return addFileToPackageVersionUnchecked(ctx, pv, pfci) + }) +} + +func addFileToPackageWrapper(fn func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error)) (*packages_model.PackageFile, error) { ctx, committer, err := db.TxContext(db.DefaultContext) if err != nil { - return nil, nil, err + return nil, err } defer committer.Close() - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) - if err != nil { - return nil, nil, err - } + pf, pb, blobCreated, err := fn(ctx) - pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci) removeBlob := false defer func() { if removeBlob { @@ -211,15 +226,15 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( }() if err != nil { removeBlob = blobCreated - return nil, nil, err + return nil, err } if err := committer.Commit(); err != nil { removeBlob = blobCreated - return nil, nil, err + return nil, err } - return pv, pf, nil + return pf, nil } // NewPackageBlob creates a package blob instance @@ -242,6 +257,12 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers return nil, nil, false, err } + return addFileToPackageVersionUnchecked(ctx, pv, pfci) +} + +func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) + pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) if err != nil { log.Error("Error inserting package blob: %v", err) @@ -333,6 +354,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p var typeSpecificSize int64 switch packageType { + case packages_model.TypeAlpine: + typeSpecificSize = setting.Packages.LimitSizeAlpine case packages_model.TypeCargo: typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeChef: @@ -424,6 +447,47 @@ func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi return nil } +// RemovePackageFileAndVersionIfUnreferenced deletes the package file and the version if there are no referenced files afterwards +func RemovePackageFileAndVersionIfUnreferenced(doer *user_model.User, pf *packages_model.PackageFile) error { + var pd *packages_model.PackageDescriptor + + if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { + if err := DeletePackageFile(ctx, pf); err != nil { + return err + } + + has, err := packages_model.HasVersionFileReferences(ctx, pf.VersionID) + if err != nil { + return err + } + if !has { + pv, err := packages_model.GetVersionByID(ctx, pf.VersionID) + if err != nil { + return err + } + + pd, err = packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + return err + } + + if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + + if pd != nil { + notification.NotifyPackageDelete(db.DefaultContext, doer, pd) + } + + return nil +} + // DeletePackageVersionAndReferences deletes the package version and its properties and files func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error { if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { diff --git a/templates/package/content/alpine.tmpl b/templates/package/content/alpine.tmpl new file mode 100644 index 0000000000000..2f695c089285e --- /dev/null +++ b/templates/package/content/alpine.tmpl @@ -0,0 +1,52 @@ +{{if eq .PackageDescriptor.Package.Type "alpine"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
/<branch>/<repository>
+

{{.locale.Tr "packages.alpine.registry.info" | Safe}}

+
+
+ + 
sudo curl  -o /etc/apk/keys/{{$.PackageDescriptor.Owner.LowerName}}@{{$.Domain}}.rsa.pub
+
+
+ +
+ 
sudo apk add {{$.PackageDescriptor.Package.Name}}={{$.PackageDescriptor.Version.Version}}
+
+
+
+ +
+
+
+ +

{{.locale.Tr "packages.alpine.repository"}}

+
+ + + + + + + + + + + + + + + +
{{.locale.Tr "packages.alpine.repository.branches"}}
{{Join .Branches ", "}}
{{.locale.Tr "packages.alpine.repository.repositories"}}
{{Join .Repositories ", "}}
{{.locale.Tr "packages.alpine.repository.architectures"}}
{{Join .Architectures ", "}}
+
+ + {{if .PackageDescriptor.Metadata.Description}} +

{{.locale.Tr "packages.about"}}

+
+ {{.PackageDescriptor.Metadata.Description}} +
+ {{end}} +{{end}} diff --git a/templates/package/metadata/alpine.tmpl b/templates/package/metadata/alpine.tmpl new file mode 100644 index 0000000000000..9011bfce10bab --- /dev/null +++ b/templates/package/metadata/alpine.tmpl @@ -0,0 +1,5 @@ +{{if eq .PackageDescriptor.Package.Type "alpine"}} + {{if .PackageDescriptor.Metadata.Maintainer}}
{{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Maintainer}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.PackageDescriptor.Metadata.License}}
{{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index b2a2fb1e5d57a..c9804e2380fe1 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
+ {{template "package/content/alpine" .}} {{template "package/content/cargo" .}} {{template "package/content/chef" .}} {{template "package/content/composer" .}} @@ -46,6 +47,7 @@ {{end}}
{{svg "octicon-calendar" 16 "gt-mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
{{svg "octicon-download" 16 "gt-mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
+ {{template "package/metadata/alpine" .}} {{template "package/metadata/cargo" .}} {{template "package/metadata/chef" .}} {{template "package/metadata/composer" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 9c89b21fcac41..4e470fb1813f5 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2226,6 +2226,7 @@ }, { "enum": [ + "alpine", "cargo", "chef", "composer", diff --git a/tests/integration/api_packages_alpine_test.go b/tests/integration/api_packages_alpine_test.go new file mode 100644 index 0000000000000..4371ed72462ba --- /dev/null +++ b/tests/integration/api_packages_alpine_test.go @@ -0,0 +1,229 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "io" + "net/http" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + alpine_module "code.gitea.io/gitea/modules/packages/alpine" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageAlpine(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "gitea-test" + packageVersion := "1.4.1-r3" + + base64AlpinePackageContent := `H4sIAAAAAAACA9ML9nT30wsKdtTLzjNJzjYuckjPLElN1DUzMUxMNTa11CsqTtQrKE1ioAAYAIGZ +iQmYBgJ02hDENjQxMTAzMzQ1MTVjMDA0MTQ1ZlAwYKADKC0uSSxSUGAYoWDm4sZZtypv75+q2fVT +POD1bKkFB22ms+g1z+H4dk7AhC3HwUSj9EbT0Rk3Dn55dHxy/K7Q+Nl/i+L7Z036ypcRvvpZuMiN +s7wbZL/klqRGGshv9Gi0qHTgTZfw3HytnJdx9c3NTRp/PHn+Z50uq2pjkilzjtpfd+uzQMw1M7cY +i9RXJasnT2M+vDXCesLK7MilJt8sGplj4xUlLMUun9SzY+phFpxWxRXa06AseV9WvzH3jtGGoL5A +vQkea+VKPj5R+Cb461tIk97qpa9nJYsJujTNl2B/J1P52H/D2rPr/j19uU8p7cMSq5tmXk51ReXl +F/Yddr9XsMpEwFKlXSPo3QSGwnCOG8y2uadjm6ui998WYXNYubjg78N3a7bnXjhrl5fB8voI++LI +1FP5W44e2xf4Ou2wrtyic1Onz7MzMV5ksuno2V/LVG4eN/15X/n2/2vJ2VV+T68aT327dOrhd6e6 +q5Y0V82Y83tdqkFa8TW2BvGCZ0ds/iibHVpzKuPcuSULO63/bNmfrnhjWqXzhMSXTb5Cv4vPaxSL +8LFMdqmxbN7+Y+Yi0ZyZhz4UxexLuHHFd1VFvk+kwvniq3P+f9rh52InWnL8Lpvedcecoh1GFSc5 +xZ9VBGex2V269HZfwxSVCvP35wQfi2xKX+lYMXtF48n1R65O2PLWpm69RdESMa79dlrTGazsZacu +MbMLeSSScPORZde76/MBV6SFJAAEAAAfiwgAAAAAAAID7VRLaxsxEN6zfoUgZ++OVq+1aUIhUDeY +pKa49FhmJdkW3ofRysXpr69220t9SCk0gZJ+IGaY56eBmbxY4/m9Q+vCUOTr1fLu4d2H7O8CEpQQ +k0y4lAClypgQoBSTQqoMGBMgMnrOXgCnIWJIVLLXCcaoib5110CSij/V7D9eCZ5p5f9o/5VkF/tf +MqUzCi+5/6Hv41Nxv/Nffu4fwRVdus4FjM7S+pFiffKNpTxnkMMsALmin5PnHgMtS8rkgvGFBPpp +c0tLKDk5HnYdto5e052PDmfRDXE0fnUh2VgucjYLU5h1g0mm5RhGNymMrtEccOfIKTTJsY/xOCyK +YqqT+74gExWbmI2VlJ6LeQUcyPFH2lh/9SBuV/wjfXPohDnw8HZKviGD/zYmCZgrgsHsk36u1Bcl +SB/8zne/0jV92/qYbKRF38X0niiemN2QxhvXDWOL+7tNGhGeYt+m22mwaR6pddGZNM8FSeRxj8PY +X7PaqdqAVlqWXHKnmQGmK43VlqNlILRilbBSMI2jV5Vbu5XGSVsDyGc7yd8B/gK2qgAIAAAfiwgA +AAAAAAID7dNNSgMxGAbg7MSCOxcu5wJOv0x+OlkU7K5QoYXqVsxMMihlKMwP1Fu48QQewCN4DfEQ +egUz4sYuFKEtFN9n870hWSSQN+7P7GrsrfNV3Y9dW5Z3bNMo0FJ+zmB9EhcJ41KS1lxJpRnxbsWi +FduBtm5sFa7C/ifOo7y5Lf2QeiHar6jTaDSbnF5Mp+fzOL/x+aJuy3g+HvGhs8JY4b3yOpMZOZEo +lRW+MEoTTw3ZwqU0INNjsAe2VPk/9b/L3/s/kIKzqOtk+IbJGTtmr+bx7WoxOUoun98frk/un14O +Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA` + content, err := base64.StdEncoding.DecodeString(base64AlpinePackageContent) + assert.NoError(t, err) + + branches := []string{"v3.16", "v3.17"} + repositories := []string{"main", "testing"} + + rootURL := fmt.Sprintf("/api/packages/%s/alpine", user.Name) + + t.Run("RepositoryKey", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", rootURL+"/key") + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "application/x-pem-file", resp.Header().Get("Content-Type")) + assert.Contains(t, resp.Body.String(), "-----BEGIN PUBLIC KEY-----") + }) + + for _, branch := range branches { + for _, repository := range repositories { + t.Run(fmt.Sprintf("[Branch:%s,Repository:%s]", branch, repository), func(t *testing.T) { + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + uploadURL := fmt.Sprintf("%s/%s/%s", rootURL, branch, repository) + + req := NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{})) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{})) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeAlpine) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.IsType(t, &alpine_module.VersionMetadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.NotEmpty(t, pfs) + assert.Condition(t, func() bool { + seen := false + expectedFilename := fmt.Sprintf("%s-%s.apk", packageName, packageVersion) + expectedCompositeKey := fmt.Sprintf("%s|%s|x86_64", branch, repository) + for _, pf := range pfs { + if pf.Name == expectedFilename && pf.CompositeKey == expectedCompositeKey { + if seen { + return false + } + seen = true + + assert.True(t, pf.IsLead) + + pfps, err := packages.GetProperties(db.DefaultContext, packages.PropertyTypeFile, pf.ID) + assert.NoError(t, err) + + for _, pfp := range pfps { + switch pfp.Name { + case alpine_module.PropertyBranch: + assert.Equal(t, branch, pfp.Value) + case alpine_module.PropertyRepository: + assert.Equal(t, repository, pfp.Value) + case alpine_module.PropertyArchitecture: + assert.Equal(t, "x86_64", pfp.Value) + } + } + } + } + return seen + }) + }) + + t.Run("APKINDEX", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository) + + req := NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Condition(t, func() bool { + br := bufio.NewReader(resp.Body) + + gzr, err := gzip.NewReader(br) + assert.NoError(t, err) + + for { + gzr.Multistream(false) + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + assert.NoError(t, err) + + if hd.Name == "APKINDEX" { + buf, err := io.ReadAll(tr) + assert.NoError(t, err) + + s := string(buf) + + assert.Contains(t, s, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n") + assert.Contains(t, s, "P:"+packageName+"\n") + assert.Contains(t, s, "V:"+packageVersion+"\n") + assert.Contains(t, s, "A:x86_64\n") + assert.Contains(t, s, "T:Gitea Test Package\n") + assert.Contains(t, s, "U:https://gitea.io/\n") + assert.Contains(t, s, "L:MIT\n") + assert.Contains(t, s, "S:1353\n") + assert.Contains(t, s, "I:4096\n") + assert.Contains(t, s, "o:gitea-test\n") + assert.Contains(t, s, "m:KN4CK3R \n") + assert.Contains(t, s, "t:1679498030\n") + + return true + } + } + + err = gzr.Reset(br) + if err == io.EOF { + break + } + assert.NoError(t, err) + } + + return false + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/%s-%s.apk", rootURL, branch, repository, packageName, packageVersion)) + MakeRequest(t, req, http.StatusOK) + }) + }) + } + } + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + for _, branch := range branches { + for _, repository := range repositories { + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/x86_64/%s-%s.apk", rootURL, branch, repository, packageName, packageVersion)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/x86_64/%s-%s.apk", rootURL, branch, repository, packageName, packageVersion)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNoContent) + + // Deleting the last file of an architecture should remove that index + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository)) + MakeRequest(t, req, http.StatusNotFound) + } + } + }) +} diff --git a/web_src/svg/gitea-alpine.svg b/web_src/svg/gitea-alpine.svg new file mode 100644 index 0000000000000..a297d95ec550e --- /dev/null +++ b/web_src/svg/gitea-alpine.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file From 8f3ff3072661ff2c7903d7480d5cc0f851cfd42c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 26 Mar 2023 13:29:40 +0000 Subject: [PATCH 2/8] Change key name. Add docs. --- docs/content/doc/packages/alpine.en-us.md | 133 +++++++++++++++++++ docs/content/doc/packages/overview.en-us.md | 1 + docs/content/doc/packages/storage.en-us.md | 2 +- modules/util/keypair.go | 16 +++ options/locale/locale_en-US.ini | 2 +- routers/api/packages/alpine/alpine.go | 24 +++- services/auth/source/oauth2/jwtsigningkey.go | 20 +-- services/packages/alpine/repository.go | 9 +- templates/package/content/alpine.tmpl | 2 +- 9 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 docs/content/doc/packages/alpine.en-us.md diff --git a/docs/content/doc/packages/alpine.en-us.md b/docs/content/doc/packages/alpine.en-us.md new file mode 100644 index 0000000000000..0b2b9a3c2ab79 --- /dev/null +++ b/docs/content/doc/packages/alpine.en-us.md @@ -0,0 +1,133 @@ +--- +date: "2023-03-25T00:00:00+00:00" +title: "Alpine Packages Repository" +slug: "packages/alpine" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Alpine" + weight: 4 + identifier: "alpine" +--- + +# Alpine Packages Repository + +Publish [Alpine](https://pkgs.alpinelinux.org/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Alpine registry, you need to use a HTTP client like `curl` to upload and a package manager like `apk` to consume packages. + +The following examples use `apk`. + +## Configuring the package registry + +To register the Alpine registry add the url to the list of known apk sources (`/etc/apk/repositories`): + +``` +https://gitea.example.com/api/packages/{owner}/alpine// +``` + +| Placeholder | Description | +| ------------ | ----------- | +| `owner` | The owner of the packages. | +| `branch` | The branch to use. | +| `repository` | The repository to use. | + +If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}): + +``` +https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/alpine// +``` + +The Alpine registry files are signed with a RSA key which must be known to apk. Download the public key and store it in `/etc/apk/keys/`: + +```shell +curl -JO https://gitea.example.com/api/packages/{owner}/alpine/key +``` + +Afterwards update the local package index: + +```shell +apk update +``` + +## Publish a package + +To publish an Alpine package (`*.apk`), perform a HTTP PUT operation with the package content in the request body. + +``` +PUT https://gitea.example.com/api/packages/{owner}/alpine/{branch}/{repository} +``` + +| Parameter | Description | +| ------------ | ----------- | +| `owner` | The owner of the package. | +| `branch` | The branch may match the release version of the OS, ex: `v3.17`. | +| `repository` | The repository can be used [to group packages](https://wiki.alpinelinux.org/wiki/Repositories) or just `main` or similar. | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/file.apk \ + https://gitea.example.com/api/packages/testuser/alpine/v3.17/main +``` + +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. +You cannot publish a file with the same name twice to a package. You must delete the existing package file first. + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `201 Created` | The package has been published. | +| `400 Bad Request` | The package name, version, branch, repository or architecture are invalid. | +| `409 Conflict` | A package file with the same combination of parameters exist already in the package. | + +## Delete a package + +To delete an Alpine package perform a HTTP DELETE operation. This will delete the package version too if there is no file left. + +``` +DELETE https://gitea.example.com/api/packages/{owner}/alpine/{branch}/{repository}/{architecture}/{filename} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `owner` | The owner of the package. | +| `branch` | The branch to use. | +| `repository` | The repository to use. | +| `architecture` | The package architecture. | +| `filename` | The file to delete. + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_token_or_password -X DELETE \ + https://gitea.example.com/api/packages/testuser/alpine/v3.17/main/test-package-1.0.0.apk +``` + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `204 No Content` | Success | +| `404 Not Found` | The package or file was not found. | + +## Install a package + +To install a package from the Alpine registry, execute the following commands: + +```shell +# use latest version +apk add {package_name} +# use specific version +apk add {package_name}={package_version} +``` diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 436514bfa7bc9..05e0a5c00cd6a 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -26,6 +26,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | +| [Alpine]({{< relref "doc/packages/alpine.en-us.md" >}}) | - | `apk` | | [Cargo]({{< relref "doc/packages/cargo.en-us.md" >}}) | Rust | `cargo` | | [Chef]({{< relref "doc/packages/chef.en-us.md" >}}) | - | `knife` | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | diff --git a/docs/content/doc/packages/storage.en-us.md b/docs/content/doc/packages/storage.en-us.md index 000752b774ab7..045bb9cd5a501 100644 --- a/docs/content/doc/packages/storage.en-us.md +++ b/docs/content/doc/packages/storage.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "Storage" - weight: 5 + weight: 2 identifier: "storage" --- diff --git a/modules/util/keypair.go b/modules/util/keypair.go index 5a3ce715a40f1..97f2d9ebca2d1 100644 --- a/modules/util/keypair.go +++ b/modules/util/keypair.go @@ -4,10 +4,13 @@ package util import ( + "crypto" "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" + + "github.com/minio/sha256-simd" ) // GenerateKeyPair generates a public and private keypair @@ -43,3 +46,16 @@ func pemBlockForPub(pub *rsa.PublicKey) (string, error) { }) return string(pubBytes), nil } + +// CreatePublicKeyFingerprint creates a fingerprint of the given key. +// The fingerprint is the sha256 sum of the PKIX structure of the key. +func CreatePublicKeyFingerprint(key crypto.PublicKey) ([]byte, error) { + bytes, err := x509.MarshalPKIXPublicKey(key) + if err != nil { + return nil, err + } + + checksum := sha256.Sum256(bytes) + + return checksum[:], nil +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3e4271ce3b749..fe26c973a1859 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3176,7 +3176,7 @@ versions.view_all = View all dependency.id = ID dependency.version = Version alpine.registry = Setup this registry by adding the url in your /etc/apk/repositories file: -alpine.registry.key = Add the registry public RSA key into the /etc/apk/keys folder to verify the index signature: +alpine.registry.key = Download the registry public RSA key into the /etc/apk/keys/ folder to verify the index signature: alpine.registry.info = Choose <branch> and <repository> from the list below. alpine.install = To install the package, run the following command: alpine.documentation = For more information on the Alpine registry, see the documentation. diff --git a/routers/api/packages/alpine/alpine.go b/routers/api/packages/alpine/alpine.go index 4e186b31f234a..5bbdeaaa97354 100644 --- a/routers/api/packages/alpine/alpine.go +++ b/routers/api/packages/alpine/alpine.go @@ -4,6 +4,9 @@ package alpine import ( + "crypto/x509" + "encoding/hex" + "encoding/pem" "errors" "fmt" "io" @@ -15,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/json" packages_module "code.gitea.io/gitea/modules/packages" alpine_module "code.gitea.io/gitea/modules/packages/alpine" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" @@ -35,9 +37,27 @@ func GetRepositoryKey(ctx *context.Context) { return } + pubPem, _ := pem.Decode([]byte(pub)) + if pubPem == nil { + apiError(ctx, http.StatusInternalServerError, "failed to decode private key pem") + return + } + + pubKey, err := x509.ParsePKIXPublicKey(pubPem.Bytes) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + fingerprint, err := util.CreatePublicKeyFingerprint(pubKey) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{ ContentType: "application/x-pem-file", - Filename: fmt.Sprintf("%s@%s.rsa.pub", ctx.Package.Owner.LowerName, setting.Domain), + Filename: fmt.Sprintf("%s@%s.rsa.pub", ctx.Package.Owner.LowerName, hex.EncodeToString(fingerprint)), }) } diff --git a/services/auth/source/oauth2/jwtsigningkey.go b/services/auth/source/oauth2/jwtsigningkey.go index 94feddbf6b6a5..f530320b23fed 100644 --- a/services/auth/source/oauth2/jwtsigningkey.go +++ b/services/auth/source/oauth2/jwtsigningkey.go @@ -24,7 +24,6 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/golang-jwt/jwt/v4" - "github.com/minio/sha256-simd" ini "gopkg.in/ini.v1" ) @@ -84,7 +83,7 @@ type rsaSingingKey struct { } func newRSASingingKey(signingMethod jwt.SigningMethod, key *rsa.PrivateKey) (rsaSingingKey, error) { - kid, err := createPublicKeyFingerprint(key.Public().(*rsa.PublicKey)) + kid, err := util.CreatePublicKeyFingerprint(key.Public().(*rsa.PublicKey)) if err != nil { return rsaSingingKey{}, err } @@ -135,7 +134,7 @@ type eddsaSigningKey struct { } func newEdDSASingingKey(signingMethod jwt.SigningMethod, key ed25519.PrivateKey) (eddsaSigningKey, error) { - kid, err := createPublicKeyFingerprint(key.Public().(ed25519.PublicKey)) + kid, err := util.CreatePublicKeyFingerprint(key.Public().(ed25519.PublicKey)) if err != nil { return eddsaSigningKey{}, err } @@ -186,7 +185,7 @@ type ecdsaSingingKey struct { } func newECDSASingingKey(signingMethod jwt.SigningMethod, key *ecdsa.PrivateKey) (ecdsaSingingKey, error) { - kid, err := createPublicKeyFingerprint(key.Public().(*ecdsa.PublicKey)) + kid, err := util.CreatePublicKeyFingerprint(key.Public().(*ecdsa.PublicKey)) if err != nil { return ecdsaSingingKey{}, err } @@ -231,19 +230,6 @@ func (key ecdsaSingingKey) PreProcessToken(token *jwt.Token) { token.Header["kid"] = key.id } -// createPublicKeyFingerprint creates a fingerprint of the given key. -// The fingerprint is the sha256 sum of the PKIX structure of the key. -func createPublicKeyFingerprint(key interface{}) ([]byte, error) { - bytes, err := x509.MarshalPKIXPublicKey(key) - if err != nil { - return nil, err - } - - checksum := sha256.Sum256(bytes) - - return checksum[:], nil -} - // CreateJWTSigningKey creates a signing key from an algorithm / key pair. func CreateJWTSigningKey(algorithm string, key interface{}) (JWTSigningKey, error) { var signingMethod jwt.SigningMethod diff --git a/services/packages/alpine/repository.go b/services/packages/alpine/repository.go index b33b7ff937bbe..a8796ea7eedbb 100644 --- a/services/packages/alpine/repository.go +++ b/services/packages/alpine/repository.go @@ -13,6 +13,7 @@ import ( "crypto/rsa" "crypto/sha1" "crypto/x509" + "encoding/hex" "encoding/pem" "errors" "fmt" @@ -27,7 +28,6 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" alpine_module "code.gitea.io/gitea/modules/packages/alpine" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" ) @@ -307,11 +307,16 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package return err } + fingerprint, err := util.CreatePublicKeyFingerprint(&privKey.PublicKey) + if err != nil { + return err + } + signedIndexContent, _ := packages_module.NewHashedBuffer(32 * 1024 * 1024) if err := writeGzipStream( signedIndexContent, - fmt.Sprintf(".SIGN.RSA.%s@%s.rsa.pub", owner.LowerName, setting.Domain), + fmt.Sprintf(".SIGN.RSA.%s@%s.rsa.pub", owner.LowerName, hex.EncodeToString(fingerprint)), sign, false, ); err != nil { diff --git a/templates/package/content/alpine.tmpl b/templates/package/content/alpine.tmpl index 2f695c089285e..ee0ba7712b2fa 100644 --- a/templates/package/content/alpine.tmpl +++ b/templates/package/content/alpine.tmpl @@ -9,7 +9,7 @@
- 
sudo curl  -o /etc/apk/keys/{{$.PackageDescriptor.Owner.LowerName}}@{{$.Domain}}.rsa.pub
+ 
curl -JO
From 500b618dbacaefe580d54f3f4bef9ea8d53173b9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 26 Mar 2023 13:31:28 +0000 Subject: [PATCH 3/8] Change test name. --- tests/integration/api_packages_alpine_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_packages_alpine_test.go b/tests/integration/api_packages_alpine_test.go index 4371ed72462ba..473dcc042495b 100644 --- a/tests/integration/api_packages_alpine_test.go +++ b/tests/integration/api_packages_alpine_test.go @@ -139,7 +139,7 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA` }) }) - t.Run("APKINDEX", func(t *testing.T) { + t.Run("Index", func(t *testing.T) { defer tests.PrintCurrentTest(t)() url := fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository) From 927eafac083c81261e5003e0eefde030fe19f5e1 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 26 Mar 2023 14:30:14 +0000 Subject: [PATCH 4/8] Fix svg. --- public/img/svg/gitea-alpine.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/img/svg/gitea-alpine.svg b/public/img/svg/gitea-alpine.svg index b2abe872690ce..1c878013ac10c 100644 --- a/public/img/svg/gitea-alpine.svg +++ b/public/img/svg/gitea-alpine.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From 827c5be3861bd1c622bc1a36ec2836b00ac06e97 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 27 Mar 2023 06:28:41 +0000 Subject: [PATCH 5/8] Fix docs ref. --- docs/content/doc/packages/alpine.en-us.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/doc/packages/alpine.en-us.md b/docs/content/doc/packages/alpine.en-us.md index 0b2b9a3c2ab79..ab80141ebf893 100644 --- a/docs/content/doc/packages/alpine.en-us.md +++ b/docs/content/doc/packages/alpine.en-us.md @@ -40,7 +40,7 @@ https://gitea.example.com/api/packages/{owner}/alpine// | `branch` | The branch to use. | | `repository` | The repository to use. | -If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}): +If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}): ``` https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/alpine// @@ -80,7 +80,7 @@ curl --user your_username:your_password_or_token \ https://gitea.example.com/api/packages/testuser/alpine/v3.17/main ``` -If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. You cannot publish a file with the same name twice to a package. You must delete the existing package file first. The server reponds with the following HTTP Status codes. From 93065f45f5adee6f9718269f4e1a4d82dab437f4 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Fri, 5 May 2023 15:46:08 -0400 Subject: [PATCH 6/8] Update docs/content/doc/usage/packages/overview.en-us.md --- docs/content/doc/usage/packages/overview.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/usage/packages/overview.en-us.md b/docs/content/doc/usage/packages/overview.en-us.md index 56340052a2d1a..37ea7ec530c04 100644 --- a/docs/content/doc/usage/packages/overview.en-us.md +++ b/docs/content/doc/usage/packages/overview.en-us.md @@ -27,7 +27,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | -| [Alpine]({{< relref "doc/packages/alpine.en-us.md" >}}) | - | `apk` | +| [Alpine]({{< relref "doc/usage/packages/alpine.en-us.md" >}}) | - | `apk` | | [Cargo]({{< relref "doc/usage/packages/cargo.en-us.md" >}}) | Rust | `cargo` | | [Chef]({{< relref "doc/usage/packages/chef.en-us.md" >}}) | - | `knife` | | [Composer]({{< relref "doc/usage/packages/composer.en-us.md" >}}) | PHP | `composer` | From 55af276d162ac000b6e9ffa12758b0dd98ae7b49 Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 12 May 2023 17:13:42 +0200 Subject: [PATCH 7/8] Update docs/content/doc/usage/packages/alpine.en-us.md --- docs/content/doc/usage/packages/alpine.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/usage/packages/alpine.en-us.md b/docs/content/doc/usage/packages/alpine.en-us.md index a8cd0211683f5..e948a378a274b 100644 --- a/docs/content/doc/usage/packages/alpine.en-us.md +++ b/docs/content/doc/usage/packages/alpine.en-us.md @@ -114,7 +114,7 @@ curl --user your_username:your_token_or_password -X DELETE \ https://gitea.example.com/api/packages/testuser/alpine/v3.17/main/test-package-1.0.0.apk ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | From 724bd25d3c92738cc3839516abb64b3676d48c46 Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 12 May 2023 17:13:50 +0200 Subject: [PATCH 8/8] Update docs/content/doc/usage/packages/alpine.en-us.md --- docs/content/doc/usage/packages/alpine.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/usage/packages/alpine.en-us.md b/docs/content/doc/usage/packages/alpine.en-us.md index e948a378a274b..aeb86093f0398 100644 --- a/docs/content/doc/usage/packages/alpine.en-us.md +++ b/docs/content/doc/usage/packages/alpine.en-us.md @@ -83,7 +83,7 @@ curl --user your_username:your_password_or_token \ If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. You cannot publish a file with the same name twice to a package. You must delete the existing package file first. -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- |