Merge pull request #67 from flatcar-linux/kai/update-packngo

Update Packet API to lastest version

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20150810074751-d8ec1a69a250
 	github.com/kylelemons/godebug v0.0.0-20150519154555-21cb3784d9bd
 	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 // indirect
-	github.com/packethost/packngo v0.0.0-20180426081943-80f62d78849d
+	github.com/packethost/packngo v0.2.1-0.20200224173249-1156d996f0d5
 	github.com/pborman/uuid v1.2.0
 	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/pin/tftp v2.1.0+incompatible
@@ -65,7 +65,7 @@ require (
 	github.com/vmware/govmomi v0.15.0
 	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect
 	go4.org v0.0.0-20180809161055-417644f6feb5 // indirect
-	golang.org/x/crypto v0.0.0-20160913211104-119f50887f8f
+	golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67
 	golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae

go.sum

@@ -94,6 +94,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.6.2 h1:8KyC64BiO8ndiGHY5DlFWWdangUPC9QHPakFRre/Ud0=
 github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.2 h1:bHM2aVXwBtBJWxHtkSrWuI4umABCUczs52eiUS9nSiw=
+github.com/hashicorp/go-retryablehttp v0.6.2/go.mod h1:gEx6HMUGxYYhJScX7W1Il64m6cc2C1mDaW3NQ9sY1FY=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
@@ -121,6 +127,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/packethost/packngo v0.0.0-20180426081943-80f62d78849d h1:Ub/9h46DrtmuxjVsLEm4+21L7gzdcTsU+q+Wyd/7JGI=
 github.com/packethost/packngo v0.0.0-20180426081943-80f62d78849d/go.mod h1:otzZQXgoO96RTzDB/Hycg0qZcXZsWJGJRSXbmEIJ+4M=
+github.com/packethost/packngo v0.2.1-0.20200224173249-1156d996f0d5 h1:zYQI6m89hXkqnR1urzsl6xMpEP525Vh4kmUB+pF5fHo=
+github.com/packethost/packngo v0.2.1-0.20200224173249-1156d996f0d5/go.mod h1:lTgI4wr0T6uAIArEQtlx1wu+n0cEy+opDJpIBAHvQ34=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
@@ -189,6 +197,8 @@ go4.org v0.0.0-20180809161055-417644f6feb5 h1:+hE86LblG4AyDgwMCLTE6FOlM9+qjHSYS+
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/crypto v0.0.0-20160913211104-119f50887f8f h1:FpKQieGQrpU4cUGLl99NjBl/gjRPADTpB270gZeWSdA=
 golang.org/x/crypto v0.0.0-20160913211104-119f50887f8f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -212,6 +222,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

kola/harness.go

@@ -79,9 +79,10 @@ var (
 	UpdatePayloadFile string
 
 	consoleChecks = []struct {
-		desc     string
-		match    *regexp.Regexp
-		skipFlag *register.Flag
+		desc        string
+		match       *regexp.Regexp
+		skipIfMatch *regexp.Regexp
+		skipFlag    *register.Flag
 	}{
 		{
 			desc:     "emergency shell",
@@ -112,8 +113,9 @@ var (
 		},
 		{
 			// https://github.com/coreos/bugs/issues/2065
-			desc:  "excessive bonding link status messages",
-			match: regexp.MustCompile("(?s:link status up for interface [^,]+, enabling it in [0-9]+ ms.*?){10}"),
+			desc:        "excessive bonding link status messages",
+			match:       regexp.MustCompile("(?s:link status up for interface [^,]+, enabling it in [0-9]+ ms.*?){3}"),
+			skipIfMatch: regexp.MustCompile("(bond.*? link status definitely up for interface)|(bond.*? first active interface up)|(bond.*? Gained carrier)|(bond.*? link becomes ready)"),
 		},
 		{
 			// https://github.com/coreos/bugs/issues/2180
@@ -578,6 +580,12 @@ func CheckConsole(output []byte, t *register.Test) []string {
 		}
 		match := check.match.FindSubmatch(output)
 		if match != nil {
+			if check.skipIfMatch != nil {
+				skipMatch := check.skipIfMatch.FindSubmatch(output)
+				if skipMatch != nil {
+					continue
+				}
+			}
 			badness := check.desc
 			if len(match) > 1 {
 				// include first subexpression

network/mockssh/mockssh.go

@@ -106,6 +106,7 @@ func NewMockClient(handler SessionHandler) *ssh.Client {
 		Auth: []ssh.AuthMethod{
 			ssh.Password(""),
 		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	}
 
 	pipe := startMockServer(handler)

network/ssh.go

@@ -128,8 +128,9 @@ func ensurePortSuffix(host string, port int) string {
 
 func (a *SSHAgent) newClient(host string, user string, auth []ssh.AuthMethod) (*ssh.Client, error) {
 	sshcfg := ssh.ClientConfig{
-		User: user,
-		Auth: auth,
+		User:            user,
+		Auth:            auth,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	}
 	addr := ensurePortSuffix(host, defaultPort)
 	tcpconn, err := a.Dial("tcp", addr)

platform/api/packet/api.go

@@ -44,7 +44,7 @@ const (
 	// Provisioning a VM is supposed to take < 8 minutes, but in practice can take longer.
 	launchTimeout       = 10 * time.Minute
 	launchPollInterval  = 30 * time.Second
-	installTimeout      = 30 * time.Minute
+	installTimeout      = 45 * time.Minute
 	installPollInterval = 5 * time.Second
 	apiRetries          = 3
 	apiRetryInterval    = 5 * time.Second
@@ -67,8 +67,8 @@ var (
 		"arm64-usr": "c2.large.arm",
 	}
 	linuxConsole = map[string]string{
-		"amd64-usr": "ttyS1,115200",
-		"arm64-usr": "ttyAMA0,115200",
+		"amd64-usr": "ttyS1,115200n8",
+		"arm64-usr": "ttyAMA0,115200n8",
 	}
 )
 
@@ -158,7 +158,7 @@ func New(opts *Options) (*API, error) {
 		return nil, fmt.Errorf("connecting to Google Storage bucket: %v", err)
 	}
 
-	client := packngo.NewClient("github.com/coreos/mantle", opts.ApiKey, nil)
+	client := packngo.NewClientWithAuth("github.com/coreos/mantle", opts.ApiKey, nil)
 
 	return &API{
 		c:      client,
@@ -168,7 +168,7 @@ func New(opts *Options) (*API, error) {
 }
 
 func (a *API) PreflightCheck() error {
-	_, _, err := a.c.Projects.Get(a.opts.Project)
+	_, _, err := a.c.Projects.Get(a.opts.Project, nil)
 	if err != nil {
 		return fmt.Errorf("querying project %v: %v", a.opts.Project, err)
 	}
@@ -210,6 +210,8 @@ func (a *API) CreateDevice(hostname string, conf *conf.Conf, console Console) (*
 	}
 	deviceID := device.ID
 
+	plog.Debugf("Created device: %q", deviceID)
+
 	if console != nil {
 		err := a.startConsole(deviceID, console)
 		consoleStarted = true
@@ -231,17 +233,21 @@ func (a *API) CreateDevice(hostname string, conf *conf.Conf, console Console) (*
 		return nil, fmt.Errorf("no public IP address found for %v", deviceID)
 	}
 
+	plog.Debugf("Device active: %q", deviceID)
+
 	err = waitForInstall(ipAddress)
 	if err != nil {
 		a.DeleteDevice(deviceID)
 		return nil, fmt.Errorf("timed out waiting for flatcar-install: %v", err)
 	}
 
+	plog.Debugf("Finished installation of device: %q", deviceID)
+
 	return device, nil
 }
 
 func (a *API) DeleteDevice(deviceID string) error {
-	_, err := a.c.Devices.Delete(deviceID)
+	_, err := a.c.Devices.Delete(deviceID, true)
 	if err != nil {
 		return fmt.Errorf("deleting device %q: %v", deviceID, err)
 	}
@@ -338,7 +344,7 @@ ExecStart=/usr/bin/curl -fo image.bin.bz2 "%v"
 # We don't verify signatures because the iPXE script isn't verified either
 # (and, in fact, is transferred over HTTP)
 
-ExecStart=/usr/bin/flatcar-install -d /dev/sda -f image.bin.bz2 %v /userdata
+ExecStart=/usr/bin/flatcar-install -s -f image.bin.bz2 %v /userdata
 
 ExecStart=/usr/bin/systemctl --no-block isolate reboot.target
 
@@ -470,7 +476,7 @@ func (a *API) createDevice(hostname, ipxeScriptURL string) (device *packngo.Devi
 		var response *packngo.Response
 		device, response, err = a.c.Devices.Create(&packngo.DeviceCreateRequest{
 			ProjectID:     a.opts.Project,
-			Facility:      a.opts.Facility,
+			Facility:      []string{a.opts.Facility},
 			Plan:          a.opts.Plan,
 			BillingCycle:  "hourly",
 			Hostname:      hostname,
@@ -481,6 +487,10 @@ func (a *API) createDevice(hostname, ipxeScriptURL string) (device *packngo.Devi
 		if err == nil || response.StatusCode != 500 {
 			return
 		}
+		plog.Debugf("Retrying to create device after failure: %q %q %q \n", device, response, err)
+		if device != nil && device.ID != "" {
+			a.DeleteDevice(device.ID)
+		}
 		if tries > 0 {
 			time.Sleep(apiRetryInterval)
 		}
@@ -539,7 +549,7 @@ func (a *API) waitForActive(deviceID string) (*packngo.Device, error) {
 	var device *packngo.Device
 	err := util.WaitUntilReady(launchTimeout, launchPollInterval, func() (bool, error) {
 		var err error
-		device, _, err = a.c.Devices.Get(deviceID)
+		device, _, err = a.c.Devices.Get(deviceID, nil)
 		if err != nil {
 			return false, fmt.Errorf("querying device: %v", err)
 		}

platform/machine/packet/cluster.go

@@ -23,6 +23,8 @@ import (
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/packet"
 	"github.com/coreos/mantle/platform/conf"
+
+	"github.com/packethost/packngo"
 )
 
 type cluster struct {
@@ -45,70 +47,86 @@ func (pc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error)
 	consolePath := filepath.Join(pc.RuntimeConf().OutputDir, "console-"+vmname+".txt")
 	var cons *console
 	var pcons packet.Console // need a nil interface value if unused
-	if pc.sshKeyID != "" {
-		// We can only read the console if Packet has our SSH key
-		f, err := os.OpenFile(consolePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	var device *packngo.Device
+	for retry := 0; retry <= 2; retry++ {
 		if err != nil {
-			return nil, err
+			plog.Infof("Retrying to provision a machine after error: %q", err)
+			if pc.sshKeyID != "" {
+				err := os.Remove(consolePath)
+				if err != nil {
+					return nil, err
+				}
+			}
 		}
-		cons = &console{
-			pc:   pc,
-			f:    f,
-			done: make(chan interface{}),
+		if pc.sshKeyID != "" {
+			// We can only read the console if Packet has our SSH key
+			f, err := os.OpenFile(consolePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+			if err != nil {
+				return nil, err
+			}
+			cons = &console{
+				pc:   pc,
+				f:    f,
+				done: make(chan interface{}),
+			}
+			pcons = cons
 		}
-		pcons = cons
-	}
 
-	// CreateDevice unconditionally closes console when done with it
-	device, err := pc.flight.api.CreateDevice(vmname, conf, pcons)
-	if err != nil {
-		return nil, err
-	}
+		// CreateDevice unconditionally closes console when done with it
+		device, err = pc.flight.api.CreateDevice(vmname, conf, pcons)
+		if err != nil {
+			continue // provisioning error
+		}
 
-	mach := &machine{
-		cluster: pc,
-		device:  device,
-		console: cons,
-	}
-	mach.publicIP = pc.flight.api.GetDeviceAddress(device, 4, true)
-	mach.privateIP = pc.flight.api.GetDeviceAddress(device, 4, false)
-	if mach.publicIP == "" || mach.privateIP == "" {
-		mach.Destroy()
-		return nil, fmt.Errorf("couldn't find IP addresses for device")
-	}
+		mach := &machine{
+			cluster: pc,
+			device:  device,
+			console: cons,
+		}
+		mach.publicIP = pc.flight.api.GetDeviceAddress(device, 4, true)
+		mach.privateIP = pc.flight.api.GetDeviceAddress(device, 4, false)
+		if mach.publicIP == "" || mach.privateIP == "" {
+			mach.Destroy()
+			err = fmt.Errorf("couldn't find IP addresses for device")
+			continue // provisioning error
+		}
 
-	dir := filepath.Join(pc.RuntimeConf().OutputDir, mach.ID())
-	if err := os.Mkdir(dir, 0777); err != nil {
-		mach.Destroy()
-		return nil, err
-	}
+		dir := filepath.Join(pc.RuntimeConf().OutputDir, mach.ID())
+		if err := os.Mkdir(dir, 0777); err != nil {
+			mach.Destroy()
+			return nil, err
+		}
+
+		if cons != nil {
+			if err := os.Rename(consolePath, filepath.Join(dir, "console.txt")); err != nil {
+				mach.Destroy()
+				return nil, err
+			}
+		}
 
-	if cons != nil {
-		if err := os.Rename(consolePath, filepath.Join(dir, "console.txt")); err != nil {
+		confPath := filepath.Join(dir, "user-data")
+		if err := conf.WriteFile(confPath); err != nil {
 			mach.Destroy()
 			return nil, err
 		}
-	}
 
-	confPath := filepath.Join(dir, "user-data")
-	if err := conf.WriteFile(confPath); err != nil {
-		mach.Destroy()
-		return nil, err
-	}
+		if mach.journal, err = platform.NewJournal(dir); err != nil {
+			mach.Destroy()
+			return nil, err
+		}
 
-	if mach.journal, err = platform.NewJournal(dir); err != nil {
-		mach.Destroy()
-		return nil, err
-	}
+		if err := platform.StartMachine(mach, mach.journal); err != nil {
+			mach.Destroy()
+			continue // provisioning error
+		}
 
-	if err := platform.StartMachine(mach, mach.journal); err != nil {
-		mach.Destroy()
-		return nil, err
-	}
+		pc.AddMach(mach)
 
-	pc.AddMach(mach)
+		return mach, nil
+
+	}
 
-	return mach, nil
+	return nil, err
 }
 
 func (pc *cluster) vmname() string {

platform/platform.go

@@ -31,7 +31,7 @@ import (
 )
 
 const (
-	sshRetries = 60
+	sshRetries = 120
 	sshTimeout = 10 * time.Second
 )