Skip to content

Commit

Permalink
Merge pull request #501 from flatcar/tormath1/scaleway
Browse files Browse the repository at this point in the history
platform: add scaleway platform
  • Loading branch information
tormath1 authored Apr 12, 2024
2 parents 346be89 + 490eacb commit d34ff9a
Show file tree
Hide file tree
Showing 79 changed files with 25,490 additions and 12 deletions.
13 changes: 12 additions & 1 deletion cmd/kola/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var (
kolaOffering string
defaultTargetBoard = sdk.DefaultBoard()
kolaArchitectures = []string{"amd64"}
kolaPlatforms = []string{"aws", "azure", "brightbox", "do", "esx", "external", "gce", "openstack", "equinixmetal", "qemu", "qemu-unpriv"}
kolaPlatforms = []string{"aws", "azure", "brightbox", "do", "esx", "external", "gce", "openstack", "equinixmetal", "qemu", "qemu-unpriv", "scaleway"}
kolaDistros = []string{"cl", "fcos", "rhcos"}
kolaChannels = []string{"alpha", "beta", "stable", "edge", "lts"}
kolaOfferings = []string{"basic", "pro"}
Expand Down Expand Up @@ -234,6 +234,16 @@ func init() {
sv(&kola.BrightboxOptions.ClientSecret, "brightbox-client-secret", "", "Brightbox client secret")
sv(&kola.BrightboxOptions.Image, "brightbox-image", "", "Brightbox image ref")
sv(&kola.BrightboxOptions.ServerType, "brightbox-server-type", "2gb.ssd", "Brightbox server type")

// Scaleway specific options
sv(&kola.ScalewayOptions.OrganizationID, "scaleway-organization-id", "", "Scaleway organization ID")
sv(&kola.ScalewayOptions.ProjectID, "scaleway-project-id", "", "Scaleway organization ID")
sv(&kola.ScalewayOptions.Region, "scaleway-region", "fr-par", "Scaleway region")
sv(&kola.ScalewayOptions.Zone, "scaleway-zone", "fr-par-1", "Scaleway region")
sv(&kola.ScalewayOptions.AccessKey, "scaleway-access-key", "", "Scaleway credentials access key")
sv(&kola.ScalewayOptions.SecretKey, "scaleway-secret-key", "", "Scaleway credentials secret key")
sv(&kola.ScalewayOptions.Image, "scaleway-image", "", "Scaleway image ID")
sv(&kola.ScalewayOptions.InstanceType, "scaleway-instance-type", "DEV1-S", "Scaleway instance type")
}

// Sync up the command line options if there is dependency
Expand All @@ -253,6 +263,7 @@ func syncOptions() error {
kola.EquinixMetalOptions.Board = board
kola.EquinixMetalOptions.GSOptions = &kola.GCEOptions
kola.BrightboxOptions.Board = board
kola.ScalewayOptions.Board = board

validateOption := func(name, item string, valid []string) error {
for _, v := range valid {
Expand Down
12 changes: 12 additions & 0 deletions cmd/ore/scaleway.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// Copyright The Mantle Authors.
// SPDX-License-Identifier: Apache-2.0

package main

import (
"github.com/flatcar/mantle/cmd/ore/scaleway"
)

func init() {
root.AddCommand(scaleway.Scaleway)
}
72 changes: 72 additions & 0 deletions cmd/ore/scaleway/create-image.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright The Mantle Authors.
// SPDX-License-Identifier: Apache-2.0

package scaleway

import (
"context"
"fmt"
"os"
"path/filepath"

"github.com/spf13/cobra"
)

const bucket = "flatcar-testing"

var (
cmdCreate = &cobra.Command{
Use: "create-image",
Short: "Create Scaleway image",
RunE: runCreate,
Example: `IMAGE_ID=$(ore scaleway \
--scaleway-access-key "${SCALEWAY_ACCESS_KEY}" \
--scaleway-secret-key "${SCALEWAY_SECRET_KEY}" \
--scaleway-organization-id "${SCALEWAY_ORGANIZATION_ID}" \
create-image --channel beta)`,
}
channel string
version string
board string
file string
)

func init() {
Scaleway.AddCommand(cmdCreate)

cmdCreate.Flags().StringVar(&channel, "channel", "stable", "Flatcar channel")
cmdCreate.Flags().StringVar(&version, "version", "current", "Flatcar version")
cmdCreate.Flags().StringVar(&board, "board", "amd64-usr", "board used for naming with default prefix and AMI architecture")
cmdCreate.Flags().StringVar(&file, "file", "flatcar_production_scaleway_image.qcow2", "path to local Flatcar image (.qcow2)")
}

func runCreate(cmd *cobra.Command, args []string) error {
if err := API.InitializeBucket(bucket); err != nil {
return fmt.Errorf("creating bucket %s: %v", bucket, err)
}

f, err := os.Open(file)
if err != nil {
return fmt.Errorf("opening Flatcar image file %s: %v", file, err)
}

defer f.Close()

key := fmt.Sprintf("%s/%s/%s/%s", channel, version, board, filepath.Base(file))
if err := API.UploadObject(f, bucket, key, true); err != nil {
return fmt.Errorf("uploading Flatcar image file %s: %v", file, err)
}

ID, err := API.CreateSnapshot(context.Background(), bucket, key)
if err != nil {
return fmt.Errorf("creating Flatcar image: %v", err)
}

if err := API.DeleteObject(bucket, key); err != nil {
return fmt.Errorf("deleting Flatcar image from s3 bucket: %s", fmt.Sprintf("s3://%s/%s", bucket, key))
}

fmt.Println(ID)

return nil
}
36 changes: 36 additions & 0 deletions cmd/ore/scaleway/gc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright The Mantle Authors.
// SPDX-License-Identifier: Apache-2.0

package scaleway

import (
"context"
"fmt"
"time"

"github.com/spf13/cobra"
)

var (
cmdGC = &cobra.Command{
Use: "gc",
Short: "GC resources in Scaleway",
Long: `Delete instances and images created over the given duration ago`,
RunE: runGC,
}

gcDuration time.Duration
)

func init() {
Scaleway.AddCommand(cmdGC)
cmdGC.Flags().DurationVar(&gcDuration, "duration", 5*time.Hour, "how old resources must be before they're considered garbage")
}

func runGC(cmd *cobra.Command, args []string) error {
if err := API.GC(context.Background(), gcDuration); err != nil {
return fmt.Errorf("running garbage collection: %w", err)
}

return nil
}
60 changes: 60 additions & 0 deletions cmd/ore/scaleway/scaleway.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright The Mantle Authors.
// SPDX-License-Identifier: Apache-2.0
package scaleway

import (
"fmt"
"os"

"github.com/coreos/pkg/capnslog"
"github.com/flatcar/mantle/cli"
"github.com/flatcar/mantle/platform"
"github.com/flatcar/mantle/platform/api/scaleway"
"github.com/spf13/cobra"
)

var (
plog = capnslog.NewPackageLogger("github.com/flatcar/mantle", "ore/scaleway")

Scaleway = &cobra.Command{
Use: "scaleway [command]",
Short: "scaleway image utilities",
}

API *scaleway.API
region string
zone string
accessKey string
secretKey string
organizationID string
projectID string
)

func init() {
cli.WrapPreRun(Scaleway, preflightCheck)
Scaleway.PersistentFlags().StringVar(&region, "scaleway-region", "fr-par", "Scaleway region")
Scaleway.PersistentFlags().StringVar(&zone, "scaleway-zone", "fr-par-1", "Scaleway region")
Scaleway.PersistentFlags().StringVar(&accessKey, "scaleway-access-key", "", "Scaleway access key")
Scaleway.PersistentFlags().StringVar(&secretKey, "scaleway-secret-key", "", "Scaleway secret key")
Scaleway.PersistentFlags().StringVar(&organizationID, "scaleway-organization-id", "", "Scaleway organization ID")
Scaleway.PersistentFlags().StringVar(&projectID, "scaleway-project-id", "", "Scaleway project ID")
}

func preflightCheck(cmd *cobra.Command, args []string) error {
api, err := scaleway.New(&scaleway.Options{
Region: region,
Zone: zone,
AccessKey: accessKey,
SecretKey: secretKey,
OrganizationID: organizationID,
ProjectID: projectID,
Options: &platform.Options{},
})
if err != nil {
fmt.Fprintf(os.Stderr, "could not create Scaleway API client: %v\n", err)
os.Exit(1)
}

API = api
return nil
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ require (
github.com/packethost/packngo v0.21.0
github.com/pborman/uuid v1.2.0
github.com/pin/tftp v2.1.0+incompatible
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.23
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace
github.com/stretchr/testify v1.9.0
Expand Down Expand Up @@ -135,6 +136,7 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

replace github.com/Microsoft/azure-vhd-utils => github.com/kinvolk/azure-vhd-utils v0.0.0-20210818134022-97083698b75f
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.23 h1:zr2sP1pxJ+iPAmZipnwz+uXmpvMEJOndD9Y+F0Dn42A=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.23/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs=
Expand Down
5 changes: 5 additions & 0 deletions kola/harness.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
esxapi "github.com/flatcar/mantle/platform/api/esx"
gcloudapi "github.com/flatcar/mantle/platform/api/gcloud"
openstackapi "github.com/flatcar/mantle/platform/api/openstack"
scalewayapi "github.com/flatcar/mantle/platform/api/scaleway"
"github.com/flatcar/mantle/platform/conf"
"github.com/flatcar/mantle/platform/machine/aws"
"github.com/flatcar/mantle/platform/machine/azure"
Expand All @@ -57,6 +58,7 @@ import (
"github.com/flatcar/mantle/platform/machine/gcloud"
"github.com/flatcar/mantle/platform/machine/openstack"
"github.com/flatcar/mantle/platform/machine/qemu"
"github.com/flatcar/mantle/platform/machine/scaleway"
"github.com/flatcar/mantle/platform/machine/unprivqemu"
"github.com/flatcar/mantle/system"
)
Expand All @@ -75,6 +77,7 @@ var (
OpenStackOptions = openstackapi.Options{Options: &Options} // glue to set platform options from main
EquinixMetalOptions = equinixmetalapi.Options{Options: &Options} // glue to set platform options from main
QEMUOptions = qemu.Options{Options: &Options} // glue to set platform options from main
ScalewayOptions = scalewayapi.Options{Options: &Options} // glue to set platform options from main

TestParallelism int //glue var to set test parallelism from main
TAPFile string // if not "", write TAP results here
Expand Down Expand Up @@ -246,6 +249,8 @@ func NewFlight(pltfrm string) (flight platform.Flight, err error) {
flight, err = qemu.NewFlight(&QEMUOptions)
case "qemu-unpriv":
flight, err = unprivqemu.NewFlight(&QEMUOptions)
case "scaleway":
flight, err = scaleway.NewFlight(&ScalewayOptions)
default:
err = fmt.Errorf("invalid platform %q", pltfrm)
}
Expand Down
4 changes: 2 additions & 2 deletions platform/api/aws/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ type API struct {
ec2 *ec2.EC2
iam *iam.IAM
marketplace *marketplacecatalog.MarketplaceCatalog
s3 *s3.S3
S3 *s3.S3
opts *Options
}

Expand Down Expand Up @@ -98,7 +98,7 @@ func New(opts *Options) (*API, error) {
ec2: ec2.New(sess),
marketplace: marketplacecatalog.New(sess),
iam: iam.New(sess),
s3: s3.New(sess),
S3: s3.New(sess),
opts: opts,
}

Expand Down
22 changes: 13 additions & 9 deletions platform/api/aws/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ func (a *API) UploadObject(r io.Reader, bucket, path string, force bool) error {

// UploadObjectExt uploads an object to S3 with more control over options.
func (a *API) UploadObjectExt(r io.Reader, bucket, path string, force bool, policy string, contentType string, max_age int) error {
s3uploader := s3manager.NewUploaderWithClient(a.s3)
s3uploader := s3manager.NewUploaderWithClient(a.S3)

if !force {
_, err := a.s3.HeadObject(&s3.HeadObjectInput{
_, err := a.S3.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
Key: &path,
})
Expand All @@ -76,8 +76,12 @@ func (a *API) UploadObjectExt(r io.Reader, bucket, path string, force bool, poli
Body: r,
Bucket: aws.String(bucket),
Key: aws.String(path),
ACL: aws.String(policy),
}

if policy != "" {
input.ACL = aws.String(policy)
}

if max_age >= 0 {
input.CacheControl = aws.String(fmt.Sprintf("max-age=%d", max_age))
}
Expand All @@ -95,7 +99,7 @@ func (a *API) UploadObjectExt(r io.Reader, bucket, path string, force bool, poli

func (a *API) DeleteObject(bucket, path string) error {
plog.Infof("Deleting s3://%v/%v", bucket, path)
_, err := a.s3.DeleteObject(&s3.DeleteObjectInput{
_, err := a.S3.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(path),
})
Expand All @@ -106,7 +110,7 @@ func (a *API) DeleteObject(bucket, path string) error {
}

func (a *API) InitializeBucket(bucket string) error {
_, err := a.s3.CreateBucket(&s3.CreateBucketInput{
_, err := a.S3.CreateBucket(&s3.CreateBucketInput{
Bucket: &bucket,
})
if err != nil {
Expand All @@ -121,7 +125,7 @@ func (a *API) InitializeBucket(bucket string) error {

// This will modify the ACL on Objects to one of the canned ACL policies
func (a *API) PutObjectAcl(bucket, path, policy string) error {
_, err := a.s3.PutObjectAcl(&s3.PutObjectAclInput{
_, err := a.S3.PutObjectAcl(&s3.PutObjectAclInput{
ACL: aws.String(policy),
Bucket: aws.String(bucket),
Key: aws.String(path),
Expand All @@ -138,7 +142,7 @@ func (a *API) CopyObject(srcBucket, srcPath, destBucket, destPath, policy string
if err != nil {
return fmt.Errorf("creating destination bucket: %v", err)
}
_, err = a.s3.CopyObject(&s3.CopyObjectInput{
_, err = a.S3.CopyObject(&s3.CopyObjectInput{
ACL: aws.String(policy),
CopySource: aws.String(url.QueryEscape(fmt.Sprintf("%s/%s", srcBucket, srcPath))),
Bucket: aws.String(destBucket),
Expand All @@ -156,7 +160,7 @@ func (a *API) CopyObject(srcBucket, srcPath, destBucket, destPath, policy string

// Copies all objects in srcBucket to destBucket with a given canned ACL policy
func (a *API) CopyBucket(srcBucket, prefix, destBucket, policy string) error {
objects, err := a.s3.ListObjects(&s3.ListObjectsInput{
objects, err := a.S3.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(srcBucket),
Prefix: aws.String(prefix),
})
Expand All @@ -183,7 +187,7 @@ func (a *API) CopyBucket(srcBucket, prefix, destBucket, policy string) error {
// TODO: bikeshed this name
// modifies the ACL of all objects of a given prefix in srcBucket to a given canned ACL policy
func (a *API) UpdateBucketObjectsACL(srcBucket, prefix, policy string) error {
objects, err := a.s3.ListObjects(&s3.ListObjectsInput{
objects, err := a.S3.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(srcBucket),
Prefix: aws.String(prefix),
})
Expand Down
Loading

0 comments on commit d34ff9a

Please sign in to comment.