diff --git a/api/objects.pb.go b/api/objects.pb.go index 5e9528e08b..8a09af9b50 100644 --- a/api/objects.pb.go +++ b/api/objects.pb.go @@ -57,6 +57,16 @@ type Node struct { Attachment *NetworkAttachment `protobuf:"bytes,7,opt,name=attachment" json:"attachment,omitempty"` // Certificate is the TLS certificate issued for the node, if any. Certificate Certificate `protobuf:"bytes,8,opt,name=certificate" json:"certificate"` + // Role is the *observed* role for this node. It differs from the + // desired role set in Node.Spec.Role because the role here is only + // updated after the Raft member list has been reconciled with the + // desired role from the spec. + // + // This field represents the current reconciled state. If an action is + // to be performed, first verify the role in the cert. This field only + // shows the privilege level that the CA would currently grant when + // issuing or renewing the node's certificate. + Role NodeRole `protobuf:"varint,9,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"` } func (m *Node) Reset() { *m = Node{} } @@ -303,6 +313,7 @@ func (m *Node) Copy() *Node { ManagerStatus: m.ManagerStatus.Copy(), Attachment: m.Attachment.Copy(), Certificate: *m.Certificate.Copy(), + Role: m.Role, } return o @@ -504,7 +515,7 @@ func (this *Node) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 12) + s := make([]string, 0, 13) s = append(s, "&api.Node{") s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n") @@ -520,6 +531,7 @@ func (this *Node) GoString() string { s = append(s, "Attachment: "+fmt.Sprintf("%#v", this.Attachment)+",\n") } s = append(s, "Certificate: "+strings.Replace(this.Certificate.GoString(), `&`, ``, 1)+",\n") + s = append(s, "Role: "+fmt.Sprintf("%#v", this.Role)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -834,6 +846,11 @@ func (m *Node) MarshalTo(data []byte) (int, error) { return 0, err } i += n10 + if m.Role != 0 { + data[i] = 0x48 + i++ + i = encodeVarintObjects(data, i, uint64(m.Role)) + } return i, nil } @@ -1451,6 +1468,9 @@ func (m *Node) Size() (n int) { } l = m.Certificate.Size() n += 1 + l + sovObjects(uint64(l)) + if m.Role != 0 { + n += 1 + sovObjects(uint64(m.Role)) + } return n } @@ -1707,6 +1727,7 @@ func (this *Node) String() string { `ManagerStatus:` + strings.Replace(fmt.Sprintf("%v", this.ManagerStatus), "ManagerStatus", "ManagerStatus", 1) + `,`, `Attachment:` + strings.Replace(fmt.Sprintf("%v", this.Attachment), "NetworkAttachment", "NetworkAttachment", 1) + `,`, `Certificate:` + strings.Replace(strings.Replace(this.Certificate.String(), "Certificate", "Certificate", 1), `&`, ``, 1) + `,`, + `Role:` + fmt.Sprintf("%v", this.Role) + `,`, `}`, }, "") return s @@ -2268,6 +2289,25 @@ func (m *Node) Unmarshal(data []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) + } + m.Role = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + m.Role |= (NodeRole(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipObjects(data[iNdEx:]) @@ -4186,78 +4226,79 @@ var ( func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) } var fileDescriptorObjects = []byte{ - // 1161 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x8f, 0x1b, 0x35, - 0x18, 0xee, 0x24, 0xb3, 0xf9, 0x78, 0xb3, 0x59, 0x81, 0xa9, 0xca, 0x34, 0x2c, 0xc9, 0x92, 0x0a, - 0x54, 0xa1, 0x2a, 0x15, 0xa5, 0xa0, 0x2d, 0xb4, 0x82, 0x7c, 0x09, 0xa2, 0x52, 0xa8, 0xdc, 0xb2, - 0x3d, 0x46, 0xde, 0x19, 0x37, 0x0c, 0x99, 0x8c, 0x47, 0xb6, 0x93, 0x2a, 0x37, 0xc4, 0x0f, 0xe0, - 0x27, 0x20, 0xce, 0xfc, 0x09, 0xae, 0x7b, 0xe0, 0xc0, 0x0d, 0x4e, 0x11, 0x9b, 0x1b, 0x37, 0x7e, - 0x02, 0xb2, 0xc7, 0x93, 0xcc, 0x2a, 0x93, 0x65, 0x2b, 0x55, 0x7b, 0xb3, 0xe3, 0xe7, 0x79, 0xde, - 0xd7, 0xaf, 0x1f, 0xbf, 0xe3, 0x40, 0x95, 0x1d, 0x7f, 0x4f, 0x5d, 0x29, 0x5a, 0x11, 0x67, 0x92, - 0x21, 0xe4, 0x31, 0x77, 0x4c, 0x79, 0x4b, 0xbc, 0x20, 0x7c, 0x32, 0xf6, 0x65, 0x6b, 0xf6, 0x41, - 0xad, 0x22, 0xe7, 0x11, 0x35, 0x80, 0x5a, 0x45, 0x44, 0xd4, 0x4d, 0x26, 0xd7, 0xa5, 0x3f, 0xa1, - 0x42, 0x92, 0x49, 0x74, 0x7b, 0x35, 0x32, 0x4b, 0x57, 0x47, 0x6c, 0xc4, 0xf4, 0xf0, 0xb6, 0x1a, - 0xc5, 0xbf, 0x36, 0x7f, 0xb3, 0xc0, 0x7e, 0x44, 0x25, 0x41, 0x9f, 0x42, 0x71, 0x46, 0xb9, 0xf0, - 0x59, 0xe8, 0x58, 0x07, 0xd6, 0xcd, 0xca, 0x9d, 0xb7, 0x5a, 0x9b, 0x91, 0x5b, 0x47, 0x31, 0xa4, - 0x63, 0x9f, 0x2c, 0x1a, 0x57, 0x70, 0xc2, 0x40, 0xf7, 0x01, 0x5c, 0x4e, 0x89, 0xa4, 0xde, 0x90, - 0x48, 0x27, 0xa7, 0xf9, 0x6f, 0x67, 0xf1, 0x9f, 0x26, 0x49, 0xe1, 0xb2, 0x21, 0xb4, 0xa5, 0x62, - 0x4f, 0x23, 0x2f, 0x61, 0xe7, 0x2f, 0xc4, 0x36, 0x84, 0xb6, 0x6c, 0xfe, 0x93, 0x07, 0xfb, 0x6b, - 0xe6, 0x51, 0x74, 0x0d, 0x72, 0xbe, 0xa7, 0x93, 0x2f, 0x77, 0x0a, 0xcb, 0x45, 0x23, 0x37, 0xe8, - 0xe1, 0x9c, 0xef, 0xa1, 0x3b, 0x60, 0x4f, 0xa8, 0x24, 0x26, 0x2d, 0x27, 0x4b, 0x58, 0x55, 0xc0, - 0xec, 0x49, 0x63, 0xd1, 0xc7, 0x60, 0xab, 0xb2, 0x9a, 0x64, 0xf6, 0xb3, 0x38, 0x2a, 0xe6, 0x93, - 0x88, 0xba, 0x09, 0x4f, 0xe1, 0x51, 0x1f, 0x2a, 0x1e, 0x15, 0x2e, 0xf7, 0x23, 0xa9, 0x2a, 0x69, - 0x6b, 0xfa, 0x8d, 0x6d, 0xf4, 0xde, 0x1a, 0x8a, 0xd3, 0x3c, 0x74, 0x1f, 0x0a, 0x42, 0x12, 0x39, - 0x15, 0xce, 0x8e, 0x56, 0xa8, 0x6f, 0x4d, 0x40, 0xa3, 0x4c, 0x0a, 0x86, 0x83, 0xbe, 0x84, 0xbd, - 0x09, 0x09, 0xc9, 0x88, 0xf2, 0xa1, 0x51, 0x29, 0x68, 0x95, 0x77, 0x32, 0xb7, 0x1e, 0x23, 0x63, - 0x21, 0x5c, 0x9d, 0xa4, 0xa7, 0xa8, 0x0f, 0x40, 0xa4, 0x24, 0xee, 0x77, 0x13, 0x1a, 0x4a, 0xa7, - 0xa8, 0x55, 0xde, 0xcd, 0xcc, 0x85, 0xca, 0x17, 0x8c, 0x8f, 0xdb, 0x2b, 0x30, 0x4e, 0x11, 0xd1, - 0x17, 0x50, 0x71, 0x29, 0x97, 0xfe, 0x73, 0xdf, 0x25, 0x92, 0x3a, 0x25, 0xad, 0xd3, 0xc8, 0xd2, - 0xe9, 0xae, 0x61, 0x66, 0x53, 0x69, 0x66, 0xf3, 0xcf, 0x1c, 0x14, 0x9f, 0x50, 0x3e, 0xf3, 0xdd, - 0x57, 0x7b, 0xdc, 0xf7, 0xce, 0x1c, 0x77, 0x66, 0x66, 0x26, 0xec, 0xc6, 0x89, 0x1f, 0x42, 0x89, - 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x9a, 0xe3, 0xce, 0x74, 0x4b, 0xdf, 0x60, 0xf0, 0x0a, 0x8d, 0xfa, - 0x50, 0x8d, 0x5d, 0x3c, 0x3c, 0x73, 0xd6, 0x07, 0x59, 0xf4, 0x6f, 0x35, 0xd0, 0x1c, 0xd2, 0xee, - 0x34, 0x35, 0x43, 0x3d, 0xa8, 0x46, 0x9c, 0xce, 0x7c, 0x36, 0x15, 0x43, 0xbd, 0x89, 0xc2, 0x85, - 0x36, 0x81, 0x77, 0x13, 0x96, 0x9a, 0x35, 0x7f, 0xce, 0x41, 0x29, 0xc9, 0x11, 0xdd, 0x35, 0xe5, - 0xb0, 0xb6, 0x27, 0x94, 0x60, 0xb5, 0x54, 0x5c, 0x89, 0xbb, 0xb0, 0x13, 0x31, 0x2e, 0x85, 0x93, - 0x3b, 0xc8, 0x6f, 0xf3, 0xec, 0x63, 0xc6, 0x65, 0x97, 0x85, 0xcf, 0xfd, 0x11, 0x8e, 0xc1, 0xe8, - 0x19, 0x54, 0x66, 0x3e, 0x97, 0x53, 0x12, 0x0c, 0xfd, 0x48, 0x38, 0x79, 0xcd, 0x7d, 0xef, 0xbc, - 0x90, 0xad, 0xa3, 0x18, 0x3f, 0x78, 0xdc, 0xd9, 0x5b, 0x2e, 0x1a, 0xb0, 0x9a, 0x0a, 0x0c, 0x46, - 0x6a, 0x10, 0x89, 0xda, 0x23, 0x28, 0xaf, 0x56, 0xd0, 0x2d, 0x80, 0x30, 0xb6, 0xe8, 0x70, 0x65, - 0x9a, 0xea, 0x72, 0xd1, 0x28, 0x1b, 0xe3, 0x0e, 0x7a, 0xb8, 0x6c, 0x00, 0x03, 0x0f, 0x21, 0xb0, - 0x89, 0xe7, 0x71, 0x6d, 0xa1, 0x32, 0xd6, 0xe3, 0xe6, 0xef, 0x3b, 0x60, 0x3f, 0x25, 0x62, 0x7c, - 0xd9, 0x6d, 0x46, 0xc5, 0xdc, 0x30, 0xdd, 0x2d, 0x00, 0x11, 0x1f, 0xa5, 0xda, 0x8e, 0xbd, 0xde, - 0x8e, 0x39, 0x60, 0xb5, 0x1d, 0x03, 0x88, 0xb7, 0x23, 0x02, 0x26, 0xb5, 0xbf, 0x6c, 0xac, 0xc7, - 0xe8, 0x06, 0x14, 0x43, 0xe6, 0x69, 0x7a, 0x41, 0xd3, 0x61, 0xb9, 0x68, 0x14, 0x54, 0x4b, 0x19, - 0xf4, 0x70, 0x41, 0x2d, 0x0d, 0x3c, 0x75, 0x6f, 0x49, 0x18, 0x32, 0x49, 0x54, 0x53, 0x12, 0xe6, - 0xfe, 0x67, 0x1a, 0xab, 0xbd, 0x86, 0x25, 0xf7, 0x36, 0xc5, 0x44, 0x47, 0xf0, 0x46, 0x92, 0x6f, - 0x5a, 0xb0, 0xf4, 0x32, 0x82, 0xc8, 0x28, 0xa4, 0x56, 0x52, 0x7d, 0xb2, 0xbc, 0xbd, 0x4f, 0xea, - 0x0a, 0x66, 0xf5, 0xc9, 0x0e, 0x54, 0x3d, 0x2a, 0x7c, 0x4e, 0x3d, 0x7d, 0x03, 0xa9, 0x03, 0x07, - 0xd6, 0xcd, 0xbd, 0x2d, 0x9f, 0x1e, 0x23, 0x42, 0xf1, 0xae, 0xe1, 0xe8, 0x19, 0x6a, 0x43, 0xc9, - 0xf8, 0x46, 0x38, 0x15, 0xed, 0xdd, 0x0b, 0xf6, 0xc7, 0x15, 0xed, 0x4c, 0x07, 0xd9, 0x7d, 0xa9, - 0x0e, 0x72, 0x0f, 0x20, 0x60, 0xa3, 0xa1, 0xc7, 0xfd, 0x19, 0xe5, 0x4e, 0x55, 0x73, 0x6b, 0x59, - 0xdc, 0x9e, 0x46, 0xe0, 0x72, 0xc0, 0x46, 0xf1, 0xb0, 0xf9, 0xa3, 0x05, 0xaf, 0x6f, 0x24, 0x85, - 0x3e, 0x82, 0xa2, 0x49, 0xeb, 0xbc, 0x47, 0x80, 0xe1, 0xe1, 0x04, 0x8b, 0xf6, 0xa1, 0xac, 0xee, - 0x08, 0x15, 0x82, 0xc6, 0xb7, 0xbf, 0x8c, 0xd7, 0x3f, 0x20, 0x07, 0x8a, 0x24, 0xf0, 0x89, 0x5a, - 0xcb, 0xeb, 0xb5, 0x64, 0xda, 0xfc, 0x29, 0x07, 0x45, 0x23, 0x76, 0xd9, 0xed, 0xdc, 0x84, 0xdd, - 0xb8, 0x59, 0x0f, 0x60, 0x37, 0x2e, 0xa7, 0xb1, 0x84, 0xfd, 0xbf, 0x45, 0xad, 0xc4, 0xf8, 0xd8, - 0x0e, 0x0f, 0xc0, 0xf6, 0x23, 0x32, 0x31, 0xad, 0x3c, 0x33, 0xf2, 0xe0, 0x71, 0xfb, 0xd1, 0x37, - 0x51, 0xec, 0xec, 0xd2, 0x72, 0xd1, 0xb0, 0xd5, 0x0f, 0x58, 0xd3, 0x9a, 0xbf, 0xec, 0x40, 0xb1, - 0x1b, 0x4c, 0x85, 0xa4, 0xfc, 0xb2, 0x0b, 0x62, 0xc2, 0x6e, 0x14, 0xa4, 0x0b, 0x45, 0xce, 0x98, - 0x1c, 0xba, 0xe4, 0xbc, 0x5a, 0x60, 0xc6, 0x64, 0xb7, 0xdd, 0xd9, 0x53, 0x44, 0xd5, 0x48, 0xe2, - 0x39, 0x2e, 0x28, 0x6a, 0x97, 0xa0, 0x67, 0x70, 0x2d, 0x69, 0xbf, 0xc7, 0x8c, 0x49, 0x21, 0x39, - 0x89, 0x86, 0x63, 0x3a, 0x57, 0xdf, 0xbc, 0xfc, 0xb6, 0x97, 0x49, 0x3f, 0x74, 0xf9, 0x5c, 0x17, - 0xea, 0x21, 0x9d, 0xe3, 0xab, 0x46, 0xa0, 0x93, 0xf0, 0x1f, 0xd2, 0xb9, 0x40, 0x9f, 0xc1, 0x3e, - 0x5d, 0xc1, 0x94, 0xe2, 0x30, 0x20, 0x13, 0xf5, 0x61, 0x19, 0xba, 0x01, 0x73, 0xc7, 0xba, 0xb7, - 0xd9, 0xf8, 0x3a, 0x4d, 0x4b, 0x7d, 0x15, 0x23, 0xba, 0x0a, 0x80, 0x04, 0x38, 0xc7, 0x01, 0x71, - 0xc7, 0x81, 0x2f, 0xd4, 0xfb, 0x33, 0xf5, 0xd8, 0x50, 0xed, 0x49, 0xe5, 0x76, 0x78, 0x4e, 0xb5, - 0x5a, 0x9d, 0x35, 0x37, 0xf5, 0x74, 0x11, 0xfd, 0x50, 0xf2, 0x39, 0x7e, 0xf3, 0x38, 0x7b, 0x15, - 0x75, 0xa0, 0x32, 0x0d, 0x55, 0xf8, 0xb8, 0x06, 0xe5, 0x8b, 0xd6, 0x00, 0x62, 0x96, 0xda, 0x79, - 0x6d, 0x06, 0xfb, 0xe7, 0x05, 0x47, 0xaf, 0x41, 0x7e, 0x4c, 0xe7, 0xb1, 0x7f, 0xb0, 0x1a, 0xa2, - 0xcf, 0x61, 0x67, 0x46, 0x82, 0x29, 0x35, 0xce, 0x79, 0x3f, 0x2b, 0x5e, 0xb6, 0x24, 0x8e, 0x89, - 0x9f, 0xe4, 0x0e, 0xad, 0xe6, 0xaf, 0x16, 0x14, 0x9e, 0x50, 0x97, 0x53, 0xf9, 0x4a, 0x1d, 0x7a, - 0x78, 0xc6, 0xa1, 0xf5, 0xec, 0xc7, 0x8b, 0x8a, 0xba, 0x61, 0xd0, 0x1a, 0x94, 0xfc, 0x50, 0x52, - 0x1e, 0x92, 0x40, 0x3b, 0xb4, 0x84, 0x57, 0xf3, 0xce, 0xfe, 0xc9, 0x69, 0xfd, 0xca, 0x5f, 0xa7, - 0xf5, 0x2b, 0xff, 0x9e, 0xd6, 0xad, 0x1f, 0x96, 0x75, 0xeb, 0x64, 0x59, 0xb7, 0xfe, 0x58, 0xd6, - 0xad, 0xbf, 0x97, 0x75, 0xeb, 0xb8, 0xa0, 0xff, 0x02, 0x7d, 0xf8, 0x5f, 0x00, 0x00, 0x00, 0xff, - 0xff, 0x38, 0xf8, 0x23, 0xac, 0x72, 0x0d, 0x00, 0x00, + // 1176 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0xcf, 0x8e, 0x1b, 0x35, + 0x1c, 0xee, 0x64, 0x67, 0x93, 0xcc, 0x2f, 0x9b, 0x15, 0x98, 0xaa, 0x4c, 0x97, 0x6d, 0xb2, 0xa4, + 0x02, 0x55, 0xa8, 0x4a, 0xa1, 0x14, 0xb4, 0x85, 0x56, 0x90, 0x7f, 0x82, 0xa8, 0x14, 0x2a, 0xb7, + 0xb4, 0xc7, 0xc8, 0x3b, 0xe3, 0x86, 0x21, 0x93, 0xf1, 0xc8, 0x76, 0x52, 0xe5, 0x86, 0x78, 0x00, + 0x5e, 0x00, 0x09, 0x71, 0xe6, 0x25, 0xb8, 0xf6, 0xc0, 0x81, 0x1b, 0x9c, 0x22, 0x9a, 0x27, 0xe0, + 0x11, 0x90, 0x3d, 0x9e, 0x64, 0x56, 0x99, 0x84, 0xad, 0x54, 0xed, 0xcd, 0x8e, 0xbf, 0xef, 0xfb, + 0xfd, 0x99, 0xcf, 0xbf, 0xf5, 0x42, 0x95, 0x9d, 0x7c, 0x4f, 0x3d, 0x29, 0x9a, 0x31, 0x67, 0x92, + 0x21, 0xe4, 0x33, 0x6f, 0x44, 0x79, 0x53, 0x3c, 0x23, 0x7c, 0x3c, 0x0a, 0x64, 0x73, 0xfa, 0xc1, + 0x41, 0x45, 0xce, 0x62, 0x6a, 0x00, 0x07, 0x15, 0x11, 0x53, 0x2f, 0xdd, 0x5c, 0x96, 0xc1, 0x98, + 0x0a, 0x49, 0xc6, 0xf1, 0x8d, 0xe5, 0xca, 0x1c, 0x5d, 0x1c, 0xb2, 0x21, 0xd3, 0xcb, 0x1b, 0x6a, + 0x95, 0xfc, 0xda, 0xf8, 0xdd, 0x02, 0xfb, 0x3e, 0x95, 0x04, 0x7d, 0x0a, 0xa5, 0x29, 0xe5, 0x22, + 0x60, 0x91, 0x6b, 0x1d, 0x59, 0xd7, 0x2a, 0x37, 0xdf, 0x6a, 0xae, 0x47, 0x6e, 0x3e, 0x4e, 0x20, + 0x6d, 0xfb, 0xf9, 0xbc, 0x7e, 0x01, 0xa7, 0x0c, 0x74, 0x07, 0xc0, 0xe3, 0x94, 0x48, 0xea, 0x0f, + 0x88, 0x74, 0x0b, 0x9a, 0x7f, 0x25, 0x8f, 0xff, 0x28, 0x4d, 0x0a, 0x3b, 0x86, 0xd0, 0x92, 0x8a, + 0x3d, 0x89, 0xfd, 0x94, 0xbd, 0x73, 0x26, 0xb6, 0x21, 0xb4, 0x64, 0xe3, 0x67, 0x1b, 0xec, 0xaf, + 0x99, 0x4f, 0xd1, 0x25, 0x28, 0x04, 0xbe, 0x4e, 0xde, 0x69, 0x17, 0x17, 0xf3, 0x7a, 0xa1, 0xdf, + 0xc5, 0x85, 0xc0, 0x47, 0x37, 0xc1, 0x1e, 0x53, 0x49, 0x4c, 0x5a, 0x6e, 0x9e, 0xb0, 0xea, 0x80, + 0xa9, 0x49, 0x63, 0xd1, 0xc7, 0x60, 0xab, 0xb6, 0x9a, 0x64, 0x0e, 0xf3, 0x38, 0x2a, 0xe6, 0xc3, + 0x98, 0x7a, 0x29, 0x4f, 0xe1, 0x51, 0x0f, 0x2a, 0x3e, 0x15, 0x1e, 0x0f, 0x62, 0xa9, 0x3a, 0x69, + 0x6b, 0xfa, 0xd5, 0x4d, 0xf4, 0xee, 0x0a, 0x8a, 0xb3, 0x3c, 0x74, 0x07, 0x8a, 0x42, 0x12, 0x39, + 0x11, 0xee, 0xae, 0x56, 0xa8, 0x6d, 0x4c, 0x40, 0xa3, 0x4c, 0x0a, 0x86, 0x83, 0xbe, 0x84, 0xfd, + 0x31, 0x89, 0xc8, 0x90, 0xf2, 0x81, 0x51, 0x29, 0x6a, 0x95, 0xb7, 0x73, 0x4b, 0x4f, 0x90, 0x89, + 0x10, 0xae, 0x8e, 0xb3, 0x5b, 0xd4, 0x03, 0x20, 0x52, 0x12, 0xef, 0xbb, 0x31, 0x8d, 0xa4, 0x5b, + 0xd2, 0x2a, 0xef, 0xe4, 0xe6, 0x42, 0xe5, 0x33, 0xc6, 0x47, 0xad, 0x25, 0x18, 0x67, 0x88, 0xe8, + 0x0b, 0xa8, 0x78, 0x94, 0xcb, 0xe0, 0x69, 0xe0, 0x11, 0x49, 0xdd, 0xb2, 0xd6, 0xa9, 0xe7, 0xe9, + 0x74, 0x56, 0x30, 0x53, 0x54, 0x96, 0x89, 0xde, 0x07, 0x9b, 0xb3, 0x90, 0xba, 0xce, 0x91, 0x75, + 0x6d, 0x7f, 0xf3, 0x67, 0xc1, 0x2c, 0xa4, 0x58, 0x23, 0x1b, 0x7f, 0x15, 0xa0, 0xf4, 0x90, 0xf2, + 0x69, 0xe0, 0xbd, 0x5a, 0x83, 0xdc, 0x3e, 0x65, 0x90, 0xdc, 0x5a, 0x4c, 0xd8, 0x35, 0x8f, 0x1c, + 0x43, 0x99, 0x46, 0x7e, 0xcc, 0x82, 0x48, 0x1a, 0x83, 0xe4, 0x16, 0xd2, 0x33, 0x18, 0xbc, 0x44, + 0xa3, 0x1e, 0x54, 0x13, 0xdf, 0x0f, 0x4e, 0xb9, 0xe3, 0x28, 0x8f, 0xfe, 0xad, 0x06, 0x9a, 0xcf, + 0xba, 0x37, 0xc9, 0xec, 0x50, 0x17, 0xaa, 0x31, 0xa7, 0xd3, 0x80, 0x4d, 0xc4, 0x40, 0x17, 0x51, + 0x3c, 0x53, 0x11, 0x78, 0x2f, 0x65, 0xa9, 0x5d, 0xe3, 0x97, 0x02, 0x94, 0xd3, 0x1c, 0xd1, 0x2d, + 0xd3, 0x0e, 0x6b, 0x73, 0x42, 0x29, 0x56, 0x4b, 0x25, 0x9d, 0xb8, 0x05, 0xbb, 0x31, 0xe3, 0x52, + 0xb8, 0x85, 0xa3, 0x9d, 0x4d, 0x2e, 0x7f, 0xc0, 0xb8, 0xec, 0xb0, 0xe8, 0x69, 0x30, 0xc4, 0x09, + 0x18, 0x3d, 0x81, 0xca, 0x34, 0xe0, 0x72, 0x42, 0xc2, 0x41, 0x10, 0x0b, 0x77, 0x47, 0x73, 0xdf, + 0xdd, 0x16, 0xb2, 0xf9, 0x38, 0xc1, 0xf7, 0x1f, 0xb4, 0xf7, 0x17, 0xf3, 0x3a, 0x2c, 0xb7, 0x02, + 0x83, 0x91, 0xea, 0xc7, 0xe2, 0xe0, 0x3e, 0x38, 0xcb, 0x13, 0x74, 0x1d, 0x20, 0x4a, 0x4c, 0x3d, + 0x58, 0x9a, 0xa6, 0xba, 0x98, 0xd7, 0x1d, 0x63, 0xf5, 0x7e, 0x17, 0x3b, 0x06, 0xd0, 0xf7, 0x11, + 0x02, 0x9b, 0xf8, 0x3e, 0xd7, 0x16, 0x72, 0xb0, 0x5e, 0x37, 0xfe, 0xd8, 0x05, 0xfb, 0x11, 0x11, + 0xa3, 0xf3, 0x1e, 0x4c, 0x2a, 0xe6, 0x9a, 0xe9, 0xae, 0x03, 0x88, 0xe4, 0x53, 0xaa, 0x72, 0xec, + 0x55, 0x39, 0xe6, 0x03, 0xab, 0x72, 0x0c, 0x20, 0x29, 0x47, 0x84, 0x4c, 0x6a, 0x7f, 0xd9, 0x58, + 0xaf, 0xd1, 0x55, 0x28, 0x45, 0xcc, 0xd7, 0xf4, 0xa2, 0xa6, 0xc3, 0x62, 0x5e, 0x2f, 0xaa, 0xeb, + 0xd6, 0xef, 0xe2, 0xa2, 0x3a, 0xea, 0xfb, 0xea, 0xa6, 0x93, 0x28, 0x62, 0x92, 0xa8, 0x31, 0x26, + 0xcc, 0xc4, 0xc8, 0x35, 0x56, 0x6b, 0x05, 0x4b, 0x6f, 0x7a, 0x86, 0x89, 0x1e, 0xc3, 0x1b, 0x69, + 0xbe, 0x59, 0xc1, 0xf2, 0xcb, 0x08, 0x22, 0xa3, 0x90, 0x39, 0xc9, 0x4c, 0x56, 0x67, 0xf3, 0x64, + 0xd5, 0x1d, 0xcc, 0x9b, 0xac, 0x6d, 0xa8, 0xfa, 0x54, 0x04, 0x9c, 0xfa, 0xfa, 0x06, 0x52, 0x17, + 0xf4, 0x20, 0xba, 0xb2, 0x4d, 0x84, 0xe2, 0x3d, 0xc3, 0xd1, 0x3b, 0xd4, 0x82, 0xb2, 0xf1, 0x8d, + 0x70, 0x2b, 0xda, 0xbb, 0x67, 0x9c, 0xa8, 0x4b, 0xda, 0xa9, 0x09, 0xb2, 0xf7, 0x52, 0x13, 0xe4, + 0x36, 0x40, 0xc8, 0x86, 0x03, 0x9f, 0x07, 0x53, 0xca, 0xdd, 0xaa, 0xe6, 0x1e, 0xe4, 0x71, 0xbb, + 0x1a, 0x81, 0x9d, 0x90, 0x0d, 0x93, 0x65, 0xe3, 0x47, 0x0b, 0x5e, 0x5f, 0x4b, 0x0a, 0x7d, 0x04, + 0x25, 0x93, 0xd6, 0xb6, 0x67, 0x83, 0xe1, 0xe1, 0x14, 0x8b, 0x0e, 0xc1, 0x51, 0x77, 0x84, 0x0a, + 0x41, 0x93, 0xdb, 0xef, 0xe0, 0xd5, 0x0f, 0xc8, 0x85, 0x12, 0x09, 0x03, 0xa2, 0xce, 0x76, 0xf4, + 0x59, 0xba, 0x6d, 0xfc, 0x54, 0x80, 0x92, 0x11, 0x3b, 0xef, 0x71, 0x6e, 0xc2, 0xae, 0xdd, 0xac, + 0xbb, 0xb0, 0x97, 0xb4, 0xd3, 0x58, 0xc2, 0xfe, 0xdf, 0xa6, 0x56, 0x12, 0x7c, 0x62, 0x87, 0xbb, + 0x60, 0x07, 0x31, 0x19, 0x9b, 0x51, 0x9e, 0x1b, 0xb9, 0xff, 0xa0, 0x75, 0xff, 0x9b, 0x38, 0x71, + 0x76, 0x79, 0x31, 0xaf, 0xdb, 0xea, 0x07, 0xac, 0x69, 0x8d, 0x5f, 0x77, 0xa1, 0xd4, 0x09, 0x27, + 0x42, 0x52, 0x7e, 0xde, 0x0d, 0x31, 0x61, 0xd7, 0x1a, 0xd2, 0x81, 0x12, 0x67, 0x4c, 0x0e, 0x3c, + 0xb2, 0xad, 0x17, 0x98, 0x31, 0xd9, 0x69, 0xb5, 0xf7, 0x15, 0x51, 0x0d, 0x92, 0x64, 0x8f, 0x8b, + 0x8a, 0xda, 0x21, 0xe8, 0x09, 0x5c, 0x4a, 0xc7, 0xef, 0x09, 0x63, 0x52, 0x48, 0x4e, 0xe2, 0xc1, + 0x88, 0xce, 0xd4, 0xdf, 0xbc, 0x9d, 0x4d, 0x6f, 0x99, 0x5e, 0xe4, 0xf1, 0x99, 0x6e, 0xd4, 0x3d, + 0x3a, 0xc3, 0x17, 0x8d, 0x40, 0x3b, 0xe5, 0xdf, 0xa3, 0x33, 0x81, 0x3e, 0x83, 0x43, 0xba, 0x84, + 0x29, 0xc5, 0x41, 0x48, 0xc6, 0xea, 0x0f, 0xcb, 0xc0, 0x0b, 0x99, 0x37, 0xd2, 0xb3, 0xcd, 0xc6, + 0x97, 0x69, 0x56, 0xea, 0xab, 0x04, 0xd1, 0x51, 0x00, 0x24, 0xc0, 0x3d, 0x09, 0x89, 0x37, 0x0a, + 0x03, 0xa1, 0x5e, 0xac, 0x99, 0xe7, 0x89, 0x1a, 0x4f, 0x2a, 0xb7, 0xe3, 0x2d, 0xdd, 0x6a, 0xb6, + 0x57, 0xdc, 0xcc, 0x63, 0x47, 0xf4, 0x22, 0xc9, 0x67, 0xf8, 0xcd, 0x93, 0xfc, 0x53, 0xd4, 0x86, + 0xca, 0x24, 0x52, 0xe1, 0x93, 0x1e, 0x38, 0x67, 0xed, 0x01, 0x24, 0x2c, 0x55, 0xf9, 0xc1, 0x14, + 0x0e, 0xb7, 0x05, 0x47, 0xaf, 0xc1, 0xce, 0x88, 0xce, 0x12, 0xff, 0x60, 0xb5, 0x44, 0x9f, 0xc3, + 0xee, 0x94, 0x84, 0x13, 0x6a, 0x9c, 0xf3, 0x5e, 0x5e, 0xbc, 0x7c, 0x49, 0x9c, 0x10, 0x3f, 0x29, + 0x1c, 0x5b, 0x8d, 0xdf, 0x2c, 0x28, 0x3e, 0xa4, 0x1e, 0xa7, 0xf2, 0x95, 0x3a, 0xf4, 0xf8, 0x94, + 0x43, 0x6b, 0xf9, 0x8f, 0x17, 0x15, 0x75, 0xcd, 0xa0, 0x07, 0x50, 0x0e, 0x22, 0x49, 0x79, 0x44, + 0x42, 0xed, 0xd0, 0x32, 0x5e, 0xee, 0xdb, 0x87, 0xcf, 0x5f, 0xd4, 0x2e, 0xfc, 0xfd, 0xa2, 0x76, + 0xe1, 0xdf, 0x17, 0x35, 0xeb, 0x87, 0x45, 0xcd, 0x7a, 0xbe, 0xa8, 0x59, 0x7f, 0x2e, 0x6a, 0xd6, + 0x3f, 0x8b, 0x9a, 0x75, 0x52, 0xd4, 0xff, 0x34, 0x7d, 0xf8, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x9f, 0x52, 0xfb, 0x4a, 0xa4, 0x0d, 0x00, 0x00, } diff --git a/api/objects.proto b/api/objects.proto index fb15285472..528d97b045 100644 --- a/api/objects.proto +++ b/api/objects.proto @@ -48,6 +48,17 @@ message Node { // Certificate is the TLS certificate issued for the node, if any. Certificate certificate = 8 [(gogoproto.nullable) = false]; + + // Role is the *observed* role for this node. It differs from the + // desired role set in Node.Spec.Role because the role here is only + // updated after the Raft member list has been reconciled with the + // desired role from the spec. + // + // This field represents the current reconciled state. If an action is + // to be performed, first verify the role in the cert. This field only + // shows the privilege level that the CA would currently grant when + // issuing or renewing the node's certificate. + NodeRole role = 9; } message Service { diff --git a/api/specs.pb.go b/api/specs.pb.go index 3ad85df737..efe7d1a147 100644 --- a/api/specs.pb.go +++ b/api/specs.pb.go @@ -112,8 +112,8 @@ func (EndpointSpec_ResolutionMode) EnumDescriptor() ([]byte, []int) { type NodeSpec struct { Annotations Annotations `protobuf:"bytes,1,opt,name=annotations" json:"annotations"` - // Role defines the role the node should have. - Role NodeRole `protobuf:"varint,2,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"` + // DesiredRole defines the role the node should have. + DesiredRole NodeRole `protobuf:"varint,2,opt,name=desired_role,json=desiredRole,proto3,enum=docker.swarmkit.v1.NodeRole" json:"desired_role,omitempty"` // Membership controls the admission of the node into the cluster. Membership NodeSpec_Membership `protobuf:"varint,3,opt,name=membership,proto3,enum=docker.swarmkit.v1.NodeSpec_Membership" json:"membership,omitempty"` // Availability allows a user to control the current scheduling status of a @@ -645,7 +645,7 @@ func (m *NodeSpec) Copy() *NodeSpec { o := &NodeSpec{ Annotations: *m.Annotations.Copy(), - Role: m.Role, + DesiredRole: m.DesiredRole, Membership: m.Membership, Availability: m.Availability, } @@ -941,7 +941,7 @@ func (this *NodeSpec) GoString() string { s := make([]string, 0, 8) s = append(s, "&api.NodeSpec{") s = append(s, "Annotations: "+strings.Replace(this.Annotations.GoString(), `&`, ``, 1)+",\n") - s = append(s, "Role: "+fmt.Sprintf("%#v", this.Role)+",\n") + s = append(s, "DesiredRole: "+fmt.Sprintf("%#v", this.DesiredRole)+",\n") s = append(s, "Membership: "+fmt.Sprintf("%#v", this.Membership)+",\n") s = append(s, "Availability: "+fmt.Sprintf("%#v", this.Availability)+",\n") s = append(s, "}") @@ -1241,10 +1241,10 @@ func (m *NodeSpec) MarshalTo(data []byte) (int, error) { return 0, err } i += n1 - if m.Role != 0 { + if m.DesiredRole != 0 { data[i] = 0x10 i++ - i = encodeVarintSpecs(data, i, uint64(m.Role)) + i = encodeVarintSpecs(data, i, uint64(m.DesiredRole)) } if m.Membership != 0 { data[i] = 0x18 @@ -2106,8 +2106,8 @@ func (m *NodeSpec) Size() (n int) { _ = l l = m.Annotations.Size() n += 1 + l + sovSpecs(uint64(l)) - if m.Role != 0 { - n += 1 + sovSpecs(uint64(m.Role)) + if m.DesiredRole != 0 { + n += 1 + sovSpecs(uint64(m.DesiredRole)) } if m.Membership != 0 { n += 1 + sovSpecs(uint64(m.Membership)) @@ -2461,7 +2461,7 @@ func (this *NodeSpec) String() string { } s := strings.Join([]string{`&NodeSpec{`, `Annotations:` + strings.Replace(strings.Replace(this.Annotations.String(), "Annotations", "Annotations", 1), `&`, ``, 1) + `,`, - `Role:` + fmt.Sprintf("%v", this.Role) + `,`, + `DesiredRole:` + fmt.Sprintf("%v", this.DesiredRole) + `,`, `Membership:` + fmt.Sprintf("%v", this.Membership) + `,`, `Availability:` + fmt.Sprintf("%v", this.Availability) + `,`, `}`, @@ -2750,9 +2750,9 @@ func (m *NodeSpec) Unmarshal(data []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field DesiredRole", wireType) } - m.Role = 0 + m.DesiredRole = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowSpecs @@ -2762,7 +2762,7 @@ func (m *NodeSpec) Unmarshal(data []byte) error { } b := data[iNdEx] iNdEx++ - m.Role |= (NodeRole(b) & 0x7F) << shift + m.DesiredRole |= (NodeRole(b) & 0x7F) << shift if b < 0x80 { break } @@ -5283,108 +5283,108 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1640 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x72, 0xdb, 0xc6, - 0x15, 0x26, 0x24, 0x8a, 0x3f, 0x07, 0x94, 0x4d, 0xed, 0xe4, 0x07, 0x66, 0x12, 0x8a, 0x66, 0xdc, - 0x54, 0x69, 0xa6, 0x72, 0xab, 0x76, 0x52, 0xa7, 0x6e, 0xa6, 0x25, 0x45, 0x56, 0x56, 0x55, 0x29, - 0x9c, 0x95, 0xe2, 0x8e, 0xaf, 0x38, 0x2b, 0x60, 0x45, 0x62, 0x04, 0x62, 0xd1, 0xdd, 0x05, 0x33, - 0xbc, 0xeb, 0x65, 0xc6, 0x17, 0x7d, 0x03, 0x5f, 0xf5, 0x19, 0xfa, 0x0e, 0xbe, 0xec, 0x65, 0x7b, - 0xa3, 0xa9, 0xf9, 0x04, 0x9d, 0xe9, 0x03, 0xb4, 0xb3, 0x8b, 0x05, 0x08, 0x26, 0x50, 0x9c, 0x99, - 0xf8, 0x6e, 0xf7, 0xe0, 0xfb, 0x0e, 0xce, 0x9e, 0xfd, 0x70, 0xce, 0x01, 0xd8, 0x22, 0xa2, 0xae, - 0xd8, 0x8f, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0xd7, 0x94, 0xef, 0x8b, 0xaf, 0x08, 0x9f, 0x5d, - 0xfb, 0x72, 0x7f, 0xfe, 0xf3, 0x96, 0x2d, 0x17, 0x11, 0x35, 0x80, 0xd6, 0x5b, 0x13, 0x36, 0x61, - 0x7a, 0xf9, 0x50, 0xad, 0x8c, 0xf5, 0x5d, 0x2f, 0xe6, 0x44, 0xfa, 0x2c, 0x7c, 0x98, 0x2e, 0x92, - 0x07, 0xdd, 0xbf, 0x96, 0xa1, 0x76, 0xc6, 0x3c, 0x7a, 0x1e, 0x51, 0x17, 0x1d, 0x81, 0x4d, 0xc2, - 0x90, 0x49, 0x0d, 0x10, 0x8e, 0xd5, 0xb1, 0xf6, 0xec, 0x83, 0xdd, 0xfd, 0x6f, 0xbf, 0x72, 0xbf, - 0xb7, 0x82, 0xf5, 0xcb, 0x2f, 0x6f, 0x76, 0x4b, 0x38, 0xcf, 0x44, 0x3f, 0x83, 0x32, 0x67, 0x01, - 0x75, 0x36, 0x3a, 0xd6, 0xde, 0x9d, 0x83, 0xf7, 0x8b, 0x3c, 0xa8, 0x97, 0x62, 0x16, 0x50, 0xac, - 0x91, 0xe8, 0x08, 0x60, 0x46, 0x67, 0x97, 0x94, 0x8b, 0xa9, 0x1f, 0x39, 0x9b, 0x9a, 0xf7, 0xe3, - 0xdb, 0x78, 0x2a, 0xd8, 0xfd, 0xd3, 0x0c, 0x8e, 0x73, 0x54, 0x74, 0x0a, 0x0d, 0x32, 0x27, 0x7e, - 0x40, 0x2e, 0xfd, 0xc0, 0x97, 0x0b, 0xa7, 0xac, 0x5d, 0x7d, 0xfc, 0x9d, 0xae, 0x7a, 0x39, 0x02, - 0x5e, 0xa3, 0x77, 0x3d, 0x80, 0xd5, 0x8b, 0xd0, 0x47, 0x50, 0x1d, 0x0d, 0xcf, 0x06, 0xc7, 0x67, - 0x47, 0xcd, 0x52, 0xeb, 0xde, 0xf3, 0x17, 0x9d, 0xb7, 0x95, 0x8f, 0x15, 0x60, 0x44, 0x43, 0xcf, - 0x0f, 0x27, 0x68, 0x0f, 0x6a, 0xbd, 0xc3, 0xc3, 0xe1, 0xe8, 0x62, 0x38, 0x68, 0x5a, 0xad, 0xd6, - 0xf3, 0x17, 0x9d, 0x77, 0xd6, 0x81, 0x3d, 0xd7, 0xa5, 0x91, 0xa4, 0x5e, 0xab, 0xfc, 0xf5, 0xdf, - 0xda, 0xa5, 0xee, 0xd7, 0x16, 0x34, 0xf2, 0x41, 0xa0, 0x8f, 0xa0, 0xd2, 0x3b, 0xbc, 0x38, 0x7e, - 0x3a, 0x6c, 0x96, 0x56, 0xf4, 0x3c, 0xa2, 0xe7, 0x4a, 0x7f, 0x4e, 0xd1, 0x03, 0xd8, 0x1a, 0xf5, - 0xbe, 0x3c, 0x1f, 0x36, 0xad, 0x55, 0x38, 0x79, 0xd8, 0x88, 0xc4, 0x42, 0xa3, 0x06, 0xb8, 0x77, - 0x7c, 0xd6, 0xdc, 0x28, 0x46, 0x0d, 0x38, 0xf1, 0x43, 0x13, 0xca, 0xab, 0x4d, 0xb0, 0xcf, 0x29, - 0x9f, 0xfb, 0xee, 0x1b, 0xd6, 0xc4, 0xa7, 0x50, 0x96, 0x44, 0x5c, 0x6b, 0x4d, 0xd8, 0xc5, 0x9a, - 0xb8, 0x20, 0xe2, 0x5a, 0xbd, 0xd4, 0xd0, 0x35, 0x5e, 0x29, 0x83, 0xd3, 0x28, 0xf0, 0x5d, 0x22, - 0xa9, 0xa7, 0x95, 0x61, 0x1f, 0xfc, 0xa8, 0x88, 0x8d, 0x33, 0x94, 0x89, 0xff, 0x49, 0x09, 0xe7, - 0xa8, 0xe8, 0x31, 0x54, 0x26, 0x01, 0xbb, 0x24, 0x81, 0xd6, 0x84, 0x7d, 0x70, 0xbf, 0xc8, 0xc9, - 0x91, 0x46, 0xac, 0x1c, 0x18, 0x0a, 0x7a, 0x04, 0x95, 0x38, 0xf2, 0x88, 0xa4, 0x4e, 0x45, 0x93, - 0x3b, 0x45, 0xe4, 0x2f, 0x35, 0xe2, 0x90, 0x85, 0x57, 0xfe, 0x04, 0x1b, 0x3c, 0x3a, 0x81, 0x5a, - 0x48, 0xe5, 0x57, 0x8c, 0x5f, 0x0b, 0xa7, 0xda, 0xd9, 0xdc, 0xb3, 0x0f, 0x3e, 0x29, 0x14, 0x63, - 0x82, 0xe9, 0x49, 0x49, 0xdc, 0xe9, 0x8c, 0x86, 0x32, 0x71, 0xd3, 0xdf, 0x70, 0x2c, 0x9c, 0x39, - 0x40, 0xbf, 0x81, 0x1a, 0x0d, 0xbd, 0x88, 0xf9, 0xa1, 0x74, 0x6a, 0xb7, 0x07, 0x32, 0x34, 0x18, - 0x95, 0x4c, 0x9c, 0x31, 0xfa, 0x15, 0x28, 0xcf, 0x98, 0x47, 0xbb, 0x0f, 0x61, 0xe7, 0x5b, 0xc9, - 0x42, 0x2d, 0xa8, 0x99, 0x64, 0x25, 0xb7, 0x5c, 0xc6, 0xd9, 0xbe, 0x7b, 0x17, 0xb6, 0xd7, 0x12, - 0xa3, 0xcb, 0x46, 0x7a, 0x5b, 0xa8, 0x07, 0x75, 0x97, 0x85, 0x92, 0xf8, 0x21, 0xe5, 0x46, 0x20, - 0x85, 0xb9, 0x3d, 0x4c, 0x41, 0x8a, 0xf5, 0xa4, 0x84, 0x57, 0x2c, 0xf4, 0x7b, 0xa8, 0x73, 0x2a, - 0x58, 0xcc, 0x5d, 0x2a, 0x8c, 0x42, 0xf6, 0x8a, 0xef, 0x38, 0x01, 0x61, 0xfa, 0xe7, 0xd8, 0xe7, - 0x54, 0xe5, 0x49, 0xe0, 0x15, 0x15, 0x3d, 0x86, 0x2a, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0xeb, 0x92, - 0x71, 0x02, 0x19, 0xb1, 0xc0, 0x77, 0x17, 0x38, 0x65, 0xa0, 0xc7, 0x50, 0x8f, 0x02, 0xe2, 0x6a, - 0xaf, 0xce, 0x96, 0xa6, 0x7f, 0x50, 0x44, 0x1f, 0xa5, 0x20, 0xbc, 0xc2, 0xa3, 0xcf, 0x00, 0x02, - 0x36, 0x19, 0x7b, 0xdc, 0x9f, 0x53, 0x6e, 0x44, 0xd2, 0x2a, 0x62, 0x0f, 0x34, 0x02, 0xd7, 0x03, - 0x36, 0x49, 0x96, 0xe8, 0xe8, 0x07, 0x29, 0x24, 0xa7, 0x8e, 0x13, 0x00, 0x92, 0x3d, 0x35, 0xfa, - 0xf8, 0xf8, 0x7b, 0xb9, 0x32, 0x37, 0x92, 0xa3, 0xa3, 0xfb, 0xd0, 0xb8, 0x62, 0xdc, 0xa5, 0x63, - 0xa3, 0xfb, 0xba, 0xd6, 0x84, 0xad, 0x6d, 0x89, 0xd0, 0xfb, 0x75, 0xa8, 0xf2, 0x38, 0x94, 0xfe, - 0x8c, 0x76, 0x4f, 0xe0, 0xed, 0x42, 0xa7, 0xe8, 0x00, 0x1a, 0xd9, 0x35, 0x8f, 0x7d, 0x4f, 0xeb, - 0xa3, 0xde, 0xbf, 0xbb, 0xbc, 0xd9, 0xb5, 0x33, 0x3d, 0x1c, 0x0f, 0xb0, 0x9d, 0x81, 0x8e, 0xbd, - 0xee, 0xbf, 0xaa, 0xb0, 0xbd, 0x26, 0x16, 0xf4, 0x16, 0x6c, 0xf9, 0x33, 0x32, 0xa1, 0x09, 0x1d, - 0x27, 0x1b, 0x34, 0x84, 0x4a, 0x40, 0x2e, 0x69, 0xa0, 0x24, 0xa3, 0xd2, 0xf6, 0xd3, 0xd7, 0xaa, - 0x6e, 0xff, 0x8f, 0x1a, 0x3f, 0x0c, 0x25, 0x5f, 0x60, 0x43, 0x46, 0x0e, 0x54, 0x5d, 0x36, 0x9b, - 0x91, 0x50, 0x95, 0x97, 0xcd, 0xbd, 0x3a, 0x4e, 0xb7, 0x08, 0x41, 0x99, 0xf0, 0x89, 0x70, 0xca, - 0xda, 0xac, 0xd7, 0xa8, 0x09, 0x9b, 0x34, 0x9c, 0x3b, 0x5b, 0xda, 0xa4, 0x96, 0xca, 0xe2, 0xf9, - 0xc9, 0x9d, 0xd7, 0xb1, 0x5a, 0x2a, 0x5e, 0x2c, 0x28, 0x77, 0xaa, 0xda, 0xa4, 0xd7, 0xe8, 0x57, - 0x50, 0x99, 0xb1, 0x38, 0x94, 0xc2, 0xa9, 0xe9, 0x60, 0xef, 0x15, 0x05, 0x7b, 0xaa, 0x10, 0xa6, - 0xfc, 0x19, 0x38, 0x7a, 0x02, 0x3b, 0x42, 0xb2, 0x68, 0x3c, 0xe1, 0xc4, 0xa5, 0xe3, 0x88, 0x72, - 0x9f, 0x79, 0xfa, 0x36, 0x6e, 0xa9, 0xa2, 0x03, 0xd3, 0xe1, 0xf1, 0x5d, 0x45, 0x3b, 0x52, 0xac, - 0x91, 0x26, 0xa1, 0x11, 0x34, 0xa2, 0x38, 0x08, 0xc6, 0x2c, 0x4a, 0x8a, 0x39, 0x68, 0x27, 0xdf, - 0x23, 0x6b, 0xa3, 0x38, 0x08, 0xbe, 0x48, 0x48, 0xd8, 0x8e, 0x56, 0x1b, 0xf4, 0x0e, 0x54, 0x26, - 0x9c, 0xc5, 0x91, 0x70, 0x6c, 0x9d, 0x0f, 0xb3, 0x43, 0x9f, 0x43, 0x55, 0x50, 0x97, 0x53, 0x29, - 0x9c, 0x86, 0x3e, 0xed, 0x87, 0x45, 0x2f, 0x39, 0xd7, 0x10, 0x4c, 0xaf, 0x28, 0xa7, 0xa1, 0x4b, - 0x71, 0xca, 0x41, 0xf7, 0x60, 0x53, 0xca, 0x85, 0xb3, 0xdd, 0xb1, 0xf6, 0x6a, 0xfd, 0xea, 0xf2, - 0x66, 0x77, 0xf3, 0xe2, 0xe2, 0x19, 0x56, 0x36, 0x55, 0xa6, 0xa6, 0x4c, 0xc8, 0x90, 0xcc, 0xa8, - 0x73, 0x47, 0xa7, 0x37, 0xdb, 0xa3, 0x67, 0x00, 0x5e, 0x28, 0xc6, 0xae, 0xfe, 0x2e, 0x9c, 0xbb, - 0xfa, 0x74, 0x9f, 0xbc, 0xfe, 0x74, 0x83, 0xb3, 0x73, 0x53, 0x6c, 0xb7, 0x97, 0x37, 0xbb, 0xf5, - 0x6c, 0x8b, 0xeb, 0x5e, 0x28, 0x92, 0x25, 0xea, 0x83, 0x3d, 0xa5, 0x24, 0x90, 0x53, 0x77, 0x4a, - 0xdd, 0x6b, 0xa7, 0x79, 0x7b, 0xed, 0x7d, 0xa2, 0x61, 0xc6, 0x43, 0x9e, 0xa4, 0x44, 0xac, 0x42, - 0x15, 0xce, 0x8e, 0xce, 0x55, 0xb2, 0x41, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x63, 0x21, 0x3d, 0x3f, - 0x74, 0x90, 0x3a, 0x32, 0xae, 0x2b, 0xcb, 0xb9, 0x32, 0xb4, 0x3e, 0x03, 0x3b, 0xa7, 0x59, 0xa5, - 0xb5, 0x6b, 0xba, 0x30, 0x9f, 0x81, 0x5a, 0x2a, 0xaf, 0x73, 0x12, 0xc4, 0xc9, 0xb0, 0x55, 0xc7, - 0xc9, 0xe6, 0xd7, 0x1b, 0x8f, 0xac, 0xd6, 0x01, 0xd8, 0xb9, 0x8b, 0x43, 0x1f, 0xc2, 0x36, 0xa7, - 0x13, 0x5f, 0x48, 0xbe, 0x18, 0x93, 0x58, 0x4e, 0x9d, 0xdf, 0x69, 0x42, 0x23, 0x35, 0xf6, 0x62, - 0x39, 0x6d, 0x8d, 0x61, 0x75, 0x7e, 0xd4, 0x01, 0x5b, 0xe5, 0x55, 0x50, 0x3e, 0xa7, 0x5c, 0x75, - 0x05, 0x15, 0x76, 0xde, 0xa4, 0xee, 0x5f, 0x50, 0xc2, 0xdd, 0xa9, 0xfe, 0x02, 0xeb, 0xd8, 0xec, - 0xd4, 0x27, 0x95, 0x8a, 0xcc, 0x7c, 0x52, 0x66, 0xdb, 0xfd, 0xaf, 0x05, 0x8d, 0x7c, 0x7b, 0x42, - 0x87, 0x49, 0x53, 0xd2, 0x47, 0xba, 0x73, 0xf0, 0xf0, 0x75, 0xed, 0x4c, 0xb7, 0x80, 0x20, 0x56, - 0xce, 0x4e, 0xd5, 0x08, 0xa9, 0xc9, 0xe8, 0x97, 0xb0, 0x15, 0x31, 0x2e, 0xd3, 0x42, 0xd0, 0x2e, - 0x2c, 0xdb, 0x8c, 0xa7, 0x25, 0x33, 0x01, 0x77, 0xa7, 0x70, 0x67, 0xdd, 0x1b, 0x7a, 0x00, 0x9b, - 0x4f, 0x8f, 0x47, 0xcd, 0x52, 0xeb, 0xbd, 0xe7, 0x2f, 0x3a, 0xef, 0xae, 0x3f, 0x7c, 0xea, 0x73, - 0x19, 0x93, 0xe0, 0x78, 0x84, 0x7e, 0x02, 0x5b, 0x83, 0xb3, 0x73, 0x8c, 0x9b, 0x56, 0x6b, 0xf7, - 0xf9, 0x8b, 0xce, 0x7b, 0xeb, 0x38, 0xf5, 0x88, 0xc5, 0xa1, 0x87, 0xd9, 0x65, 0x36, 0x55, 0xfd, - 0x7d, 0x03, 0x6c, 0x53, 0x1f, 0xdf, 0xec, 0x54, 0xf5, 0x5b, 0xd8, 0x4e, 0x5a, 0x4e, 0xaa, 0xfa, - 0x8d, 0xd7, 0x76, 0x9e, 0x46, 0x42, 0x30, 0x77, 0x7c, 0x1f, 0x1a, 0x7e, 0x34, 0xff, 0x74, 0x4c, - 0x43, 0x72, 0x19, 0x98, 0x01, 0xab, 0x86, 0x6d, 0x65, 0x1b, 0x26, 0x26, 0xf5, 0xc9, 0xf9, 0xa1, - 0xa4, 0x3c, 0x34, 0xa3, 0x53, 0x0d, 0x67, 0x7b, 0xf4, 0x39, 0x94, 0xfd, 0x88, 0xcc, 0x4c, 0xbb, - 0x2c, 0x3c, 0xc1, 0xf1, 0xa8, 0x77, 0x6a, 0x34, 0xd8, 0xaf, 0x2d, 0x6f, 0x76, 0xcb, 0xca, 0x80, - 0x35, 0x0d, 0xb5, 0xd3, 0x8e, 0xa5, 0xde, 0xa4, 0x2b, 0x68, 0x0d, 0xe7, 0x2c, 0xdd, 0xff, 0x95, - 0xc1, 0x3e, 0x0c, 0x62, 0x21, 0x4d, 0x1f, 0x78, 0x63, 0x79, 0x7b, 0x06, 0x3b, 0x44, 0xcf, 0xe0, - 0x24, 0x54, 0x45, 0x55, 0x4f, 0x02, 0x26, 0x77, 0x0f, 0x0a, 0xdd, 0x65, 0xe0, 0x64, 0x6a, 0xe8, - 0x57, 0x94, 0x4f, 0xc7, 0xc2, 0x4d, 0xf2, 0x8d, 0x27, 0xe8, 0x1c, 0xb6, 0x19, 0x77, 0xa7, 0x54, - 0xc8, 0xa4, 0x0e, 0x9b, 0x99, 0xb5, 0xf0, 0x6f, 0xe6, 0x8b, 0x3c, 0xd0, 0x14, 0xa1, 0x24, 0xda, - 0x75, 0x1f, 0xe8, 0x11, 0x94, 0x39, 0xb9, 0x4a, 0xa7, 0x9a, 0x42, 0x7d, 0x63, 0x72, 0x25, 0xd7, - 0x5c, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48, 0x77, 0x4a, 0xb9, 0xb9, 0xa7, 0xc2, - 0x23, 0x0e, 0x32, 0xd4, 0x9a, 0x97, 0x1c, 0x1b, 0x9d, 0x40, 0xdd, 0x25, 0xa9, 0xd2, 0x2a, 0xb7, - 0xb7, 0xa0, 0xc3, 0x9e, 0x71, 0xd1, 0x54, 0x2e, 0x96, 0x37, 0xbb, 0xb5, 0xd4, 0x82, 0x6b, 0x2e, - 0x31, 0xca, 0x3b, 0x81, 0x6d, 0x35, 0xe0, 0x8f, 0x3d, 0x7a, 0x45, 0xe2, 0x40, 0x0a, 0xdd, 0x2d, - 0x6f, 0x29, 0xaa, 0x6a, 0xd6, 0x1c, 0x18, 0x9c, 0x89, 0xab, 0x21, 0x73, 0x36, 0xf4, 0x27, 0xd8, - 0xa1, 0xa1, 0xcb, 0x17, 0x5a, 0x67, 0x69, 0x84, 0xb5, 0xdb, 0x0f, 0x3b, 0xcc, 0xc0, 0x6b, 0x87, - 0x6d, 0xd2, 0x6f, 0xd8, 0xbb, 0x3e, 0x40, 0xd2, 0xa6, 0xde, 0xac, 0xfe, 0x10, 0x94, 0x3d, 0x22, - 0x89, 0x96, 0x5c, 0x03, 0xeb, 0x75, 0xff, 0xfd, 0x97, 0xaf, 0xda, 0xa5, 0x7f, 0xbe, 0x6a, 0x97, - 0xfe, 0xf3, 0xaa, 0x6d, 0xfd, 0x65, 0xd9, 0xb6, 0x5e, 0x2e, 0xdb, 0xd6, 0x3f, 0x96, 0x6d, 0xeb, - 0xdf, 0xcb, 0xb6, 0x75, 0x59, 0xd1, 0x3f, 0xec, 0xbf, 0xf8, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xab, 0x02, 0x96, 0x72, 0x0f, 0x10, 0x00, 0x00, + // 1647 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0xe3, 0xc6, + 0x15, 0x16, 0x6d, 0x59, 0x3f, 0x87, 0xf2, 0xae, 0x3c, 0xc8, 0x0f, 0x57, 0x49, 0x64, 0xad, 0xb2, + 0x4d, 0x9d, 0x06, 0xf5, 0xa2, 0x6e, 0x91, 0x6e, 0xba, 0x0d, 0x5a, 0xc9, 0x52, 0xbd, 0xae, 0x6b, + 0x47, 0x18, 0x3b, 0x5b, 0xec, 0x95, 0x30, 0x26, 0xc7, 0x12, 0x61, 0x8a, 0xc3, 0xce, 0x0c, 0x15, + 0xf8, 0xae, 0x97, 0xc1, 0x5e, 0xf4, 0x0d, 0x7c, 0xd5, 0x67, 0xe8, 0x3b, 0xec, 0x65, 0x2f, 0xdb, + 0x1b, 0xa3, 0xab, 0x27, 0x28, 0xd0, 0x07, 0x68, 0x31, 0xc3, 0x21, 0x45, 0x25, 0x74, 0x76, 0x81, + 0xfa, 0x6e, 0xe6, 0xf0, 0xfb, 0x0e, 0xcf, 0x9c, 0xf9, 0x78, 0xce, 0x21, 0xd8, 0x22, 0xa2, 0xae, + 0xd8, 0x8d, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x97, 0x94, 0xef, 0x8a, 0x6f, 0x08, 0x9f, 0x5d, + 0xfa, 0x72, 0x77, 0xfe, 0xb3, 0x96, 0x2d, 0xaf, 0x22, 0x6a, 0x00, 0xad, 0x77, 0x26, 0x6c, 0xc2, + 0xf4, 0xf2, 0xb1, 0x5a, 0x19, 0xeb, 0xfb, 0x5e, 0xcc, 0x89, 0xf4, 0x59, 0xf8, 0x38, 0x5d, 0x24, + 0x0f, 0xba, 0xd7, 0x65, 0xa8, 0x9d, 0x30, 0x8f, 0x9e, 0x46, 0xd4, 0x45, 0x07, 0x60, 0x93, 0x30, + 0x64, 0x52, 0x03, 0x84, 0x63, 0x75, 0xac, 0x1d, 0x7b, 0x6f, 0x7b, 0xf7, 0xfb, 0xaf, 0xdc, 0xed, + 0x2d, 0x61, 0xfd, 0xf2, 0xab, 0x9b, 0xed, 0x12, 0xce, 0x33, 0xd1, 0x6f, 0xa0, 0xe1, 0x51, 0xe1, + 0x73, 0xea, 0x8d, 0x39, 0x0b, 0xa8, 0xb3, 0xd6, 0xb1, 0x76, 0xee, 0xed, 0x7d, 0x58, 0xe4, 0x49, + 0xbd, 0x1c, 0xb3, 0x80, 0x62, 0xdb, 0x30, 0xd4, 0x06, 0x1d, 0x00, 0xcc, 0xe8, 0xec, 0x9c, 0x72, + 0x31, 0xf5, 0x23, 0x67, 0x5d, 0xd3, 0x7f, 0x7c, 0x1b, 0x5d, 0xc5, 0xbe, 0x7b, 0x9c, 0xc1, 0x71, + 0x8e, 0x8a, 0x8e, 0xa1, 0x41, 0xe6, 0xc4, 0x0f, 0xc8, 0xb9, 0x1f, 0xf8, 0xf2, 0xca, 0x29, 0x6b, + 0x57, 0x9f, 0xfe, 0xa0, 0xab, 0x5e, 0x8e, 0x80, 0x57, 0xe8, 0x5d, 0x0f, 0x60, 0xf9, 0x22, 0xf4, + 0x09, 0x54, 0x47, 0xc3, 0x93, 0xc1, 0xe1, 0xc9, 0x41, 0xb3, 0xd4, 0x7a, 0xf0, 0xf2, 0xba, 0xf3, + 0xae, 0xf2, 0xb1, 0x04, 0x8c, 0x68, 0xe8, 0xf9, 0xe1, 0x04, 0xed, 0x40, 0xad, 0xb7, 0xbf, 0x3f, + 0x1c, 0x9d, 0x0d, 0x07, 0x4d, 0xab, 0xd5, 0x7a, 0x79, 0xdd, 0x79, 0x6f, 0x15, 0xd8, 0x73, 0x5d, + 0x1a, 0x49, 0xea, 0xb5, 0xca, 0xdf, 0xfe, 0xb5, 0x5d, 0xea, 0x7e, 0x6b, 0x41, 0x23, 0x1f, 0x04, + 0xfa, 0x04, 0x2a, 0xbd, 0xfd, 0xb3, 0xc3, 0xe7, 0xc3, 0x66, 0x69, 0x49, 0xcf, 0x23, 0x7a, 0xae, + 0xf4, 0xe7, 0x14, 0x3d, 0x82, 0x8d, 0x51, 0xef, 0xeb, 0xd3, 0x61, 0xd3, 0x5a, 0x86, 0x93, 0x87, + 0x8d, 0x48, 0x2c, 0x34, 0x6a, 0x80, 0x7b, 0x87, 0x27, 0xcd, 0xb5, 0x62, 0xd4, 0x80, 0x13, 0x3f, + 0x34, 0xa1, 0xbc, 0x5e, 0x07, 0xfb, 0x94, 0xf2, 0xb9, 0xef, 0xde, 0xb1, 0x44, 0x3e, 0x87, 0xb2, + 0x24, 0xe2, 0x52, 0x4b, 0xc3, 0x2e, 0x96, 0xc6, 0x19, 0x11, 0x97, 0xea, 0xa5, 0x86, 0xae, 0xf1, + 0x4a, 0x19, 0x9c, 0x46, 0x81, 0xef, 0x12, 0x49, 0x3d, 0xad, 0x0c, 0x7b, 0xef, 0x47, 0x45, 0x6c, + 0x9c, 0xa1, 0x4c, 0xfc, 0xcf, 0x4a, 0x38, 0x47, 0x45, 0x4f, 0xa1, 0x32, 0x09, 0xd8, 0x39, 0x09, + 0xb4, 0x26, 0xec, 0xbd, 0x87, 0x45, 0x4e, 0x0e, 0x34, 0x62, 0xe9, 0xc0, 0x50, 0xd0, 0x13, 0xa8, + 0xc4, 0x91, 0x47, 0x24, 0x75, 0x2a, 0x9a, 0xdc, 0x29, 0x22, 0x7f, 0xad, 0x11, 0xfb, 0x2c, 0xbc, + 0xf0, 0x27, 0xd8, 0xe0, 0xd1, 0x11, 0xd4, 0x42, 0x2a, 0xbf, 0x61, 0xfc, 0x52, 0x38, 0xd5, 0xce, + 0xfa, 0x8e, 0xbd, 0xf7, 0x59, 0xa1, 0x18, 0x13, 0x4c, 0x4f, 0x4a, 0xe2, 0x4e, 0x67, 0x34, 0x94, + 0x89, 0x9b, 0xfe, 0x9a, 0x63, 0xe1, 0xcc, 0x01, 0xfa, 0x35, 0xd4, 0x68, 0xe8, 0x45, 0xcc, 0x0f, + 0xa5, 0x53, 0xbb, 0x3d, 0x90, 0xa1, 0xc1, 0xa8, 0x64, 0xe2, 0x8c, 0xd1, 0xaf, 0x40, 0x79, 0xc6, + 0x3c, 0xda, 0x7d, 0x0c, 0x5b, 0xdf, 0x4b, 0x16, 0x6a, 0x41, 0xcd, 0x24, 0x2b, 0xb9, 0xe5, 0x32, + 0xce, 0xf6, 0xdd, 0xfb, 0xb0, 0xb9, 0x92, 0x98, 0xee, 0x5f, 0xca, 0x50, 0x4b, 0x6f, 0x0b, 0xf5, + 0xa0, 0xee, 0xb2, 0x50, 0x12, 0x3f, 0xa4, 0xdc, 0x08, 0xa4, 0x30, 0xb7, 0xfb, 0x29, 0x48, 0xb1, + 0x9e, 0x95, 0xf0, 0x92, 0x85, 0x7e, 0x07, 0x75, 0x4e, 0x05, 0x8b, 0xb9, 0x4b, 0x85, 0x51, 0xc8, + 0x4e, 0xf1, 0x1d, 0x27, 0x20, 0x4c, 0xff, 0x14, 0xfb, 0x9c, 0xaa, 0x3c, 0x09, 0xbc, 0xa4, 0xa2, + 0xa7, 0x50, 0xe5, 0x54, 0x48, 0xc2, 0xe5, 0x0f, 0x5d, 0x32, 0x4e, 0x20, 0x23, 0x16, 0xf8, 0xee, + 0x15, 0x4e, 0x19, 0xe8, 0x29, 0xd4, 0xa3, 0x80, 0xb8, 0xda, 0xab, 0xb3, 0xa1, 0xe9, 0x1f, 0x15, + 0xd1, 0x47, 0x29, 0x08, 0x2f, 0xf1, 0xe8, 0x0b, 0x80, 0x80, 0x4d, 0xc6, 0x1e, 0xf7, 0xe7, 0x94, + 0x1b, 0x91, 0xb4, 0x8a, 0xd8, 0x03, 0x8d, 0xc0, 0xf5, 0x80, 0x4d, 0x92, 0x25, 0x3a, 0xf8, 0xbf, + 0x14, 0x92, 0x53, 0xc7, 0x11, 0x00, 0xc9, 0x9e, 0x1a, 0x7d, 0x7c, 0xfa, 0x56, 0xae, 0xcc, 0x8d, + 0xe4, 0xe8, 0xe8, 0x21, 0x34, 0x2e, 0x18, 0x77, 0xe9, 0xd8, 0xe8, 0xbe, 0xae, 0x35, 0x61, 0x6b, + 0x5b, 0x22, 0xf4, 0x7e, 0x1d, 0xaa, 0x3c, 0x0e, 0xa5, 0x3f, 0xa3, 0xdd, 0x23, 0x78, 0xb7, 0xd0, + 0x29, 0xda, 0x83, 0x46, 0x76, 0xcd, 0x63, 0xdf, 0xd3, 0xfa, 0xa8, 0xf7, 0xef, 0x2f, 0x6e, 0xb6, + 0xed, 0x4c, 0x0f, 0x87, 0x03, 0x6c, 0x67, 0xa0, 0x43, 0xaf, 0xfb, 0xcf, 0x2a, 0x6c, 0xae, 0x88, + 0x05, 0xbd, 0x03, 0x1b, 0xfe, 0x8c, 0x4c, 0x68, 0x42, 0xc7, 0xc9, 0x06, 0x0d, 0xa1, 0x12, 0x90, + 0x73, 0x1a, 0x28, 0xc9, 0xa8, 0xb4, 0xfd, 0xf4, 0x8d, 0xaa, 0xdb, 0xfd, 0x83, 0xc6, 0x0f, 0x43, + 0xc9, 0xaf, 0xb0, 0x21, 0x23, 0x07, 0xaa, 0x2e, 0x9b, 0xcd, 0x48, 0xa8, 0xca, 0xcb, 0xfa, 0x4e, + 0x1d, 0xa7, 0x5b, 0x84, 0xa0, 0x4c, 0xf8, 0x44, 0x38, 0x65, 0x6d, 0xd6, 0x6b, 0xd4, 0x84, 0x75, + 0x1a, 0xce, 0x9d, 0x0d, 0x6d, 0x52, 0x4b, 0x65, 0xf1, 0xfc, 0xe4, 0xce, 0xeb, 0x58, 0x2d, 0x15, + 0x2f, 0x16, 0x94, 0x3b, 0x55, 0x6d, 0xd2, 0x6b, 0xf4, 0x4b, 0xa8, 0xcc, 0x58, 0x1c, 0x4a, 0xe1, + 0xd4, 0x74, 0xb0, 0x0f, 0x8a, 0x82, 0x3d, 0x56, 0x08, 0x53, 0xfe, 0x0c, 0x1c, 0x3d, 0x83, 0x2d, + 0x21, 0x59, 0x34, 0x9e, 0x70, 0xe2, 0xd2, 0x71, 0x44, 0xb9, 0xcf, 0x3c, 0x7d, 0x1b, 0xb7, 0x54, + 0xd1, 0x81, 0x69, 0xf8, 0xf8, 0xbe, 0xa2, 0x1d, 0x28, 0xd6, 0x48, 0x93, 0xd0, 0x08, 0x1a, 0x51, + 0x1c, 0x04, 0x63, 0x16, 0x25, 0xc5, 0x1c, 0xb4, 0x93, 0xb7, 0xc8, 0xda, 0x28, 0x0e, 0x82, 0xaf, + 0x12, 0x12, 0xb6, 0xa3, 0xe5, 0x06, 0xbd, 0x07, 0x95, 0x09, 0x67, 0x71, 0x24, 0x1c, 0x5b, 0xe7, + 0xc3, 0xec, 0xd0, 0x97, 0x50, 0x15, 0xd4, 0xe5, 0x54, 0x0a, 0xa7, 0xa1, 0x4f, 0xfb, 0x71, 0xd1, + 0x4b, 0x4e, 0x35, 0x04, 0xd3, 0x0b, 0xca, 0x69, 0xe8, 0x52, 0x9c, 0x72, 0xd0, 0x03, 0x58, 0x97, + 0xf2, 0xca, 0xd9, 0xec, 0x58, 0x3b, 0xb5, 0x7e, 0x75, 0x71, 0xb3, 0xbd, 0x7e, 0x76, 0xf6, 0x02, + 0x2b, 0x9b, 0x2a, 0x53, 0x53, 0x26, 0x64, 0x48, 0x66, 0xd4, 0xb9, 0xa7, 0xd3, 0x9b, 0xed, 0xd1, + 0x0b, 0x00, 0x2f, 0x14, 0x63, 0x57, 0x7f, 0x17, 0xce, 0x7d, 0x7d, 0xba, 0xcf, 0xde, 0x7c, 0xba, + 0xc1, 0xc9, 0xa9, 0x29, 0xb6, 0x9b, 0x8b, 0x9b, 0xed, 0x7a, 0xb6, 0xc5, 0x75, 0x2f, 0x14, 0xc9, + 0x12, 0xf5, 0xc1, 0x9e, 0x52, 0x12, 0xc8, 0xa9, 0x3b, 0xa5, 0xee, 0xa5, 0xd3, 0xbc, 0xbd, 0xf6, + 0x3e, 0xd3, 0x30, 0xe3, 0x21, 0x4f, 0x52, 0x22, 0x56, 0xa1, 0x0a, 0x67, 0x4b, 0xe7, 0x2a, 0xd9, + 0xa0, 0x8f, 0x00, 0x58, 0x44, 0xc3, 0xb1, 0x90, 0x9e, 0x1f, 0x3a, 0x48, 0x1d, 0x19, 0xd7, 0x95, + 0xe5, 0x54, 0x19, 0x5a, 0x5f, 0x80, 0x9d, 0xd3, 0xac, 0xd2, 0xda, 0x25, 0xbd, 0x32, 0x9f, 0x81, + 0x5a, 0x2a, 0xaf, 0x73, 0x12, 0xc4, 0xc9, 0xcc, 0x55, 0xc7, 0xc9, 0xe6, 0x57, 0x6b, 0x4f, 0xac, + 0xd6, 0x1e, 0xd8, 0xb9, 0x8b, 0x43, 0x1f, 0xc3, 0x26, 0xa7, 0x13, 0x5f, 0x48, 0x7e, 0x35, 0x26, + 0xb1, 0x9c, 0x3a, 0xbf, 0xd5, 0x84, 0x46, 0x6a, 0xec, 0xc5, 0x72, 0xda, 0x1a, 0xc3, 0xf2, 0xfc, + 0xa8, 0x03, 0xb6, 0xca, 0xab, 0xa0, 0x7c, 0x4e, 0xb9, 0xea, 0x0a, 0x2a, 0xec, 0xbc, 0x49, 0xdd, + 0xbf, 0xa0, 0x84, 0xbb, 0x53, 0xfd, 0x05, 0xd6, 0xb1, 0xd9, 0xa9, 0x4f, 0x2a, 0x15, 0x99, 0xf9, + 0xa4, 0xcc, 0xb6, 0xfb, 0x1f, 0x0b, 0x1a, 0xf9, 0xf6, 0x84, 0xf6, 0x93, 0xa6, 0xa4, 0x8f, 0x74, + 0x6f, 0xef, 0xf1, 0x9b, 0xda, 0x99, 0x6e, 0x01, 0x41, 0xac, 0x9c, 0x1d, 0xab, 0x49, 0x52, 0x93, + 0xd1, 0x2f, 0x60, 0x23, 0x62, 0x5c, 0xa6, 0x85, 0xa0, 0x5d, 0x58, 0xb6, 0x19, 0x4f, 0x4b, 0x66, + 0x02, 0xee, 0x4e, 0xe1, 0xde, 0xaa, 0x37, 0xf4, 0x08, 0xd6, 0x9f, 0x1f, 0x8e, 0x9a, 0xa5, 0xd6, + 0x07, 0x2f, 0xaf, 0x3b, 0xef, 0xaf, 0x3e, 0x7c, 0xee, 0x73, 0x19, 0x93, 0xe0, 0x70, 0x84, 0x7e, + 0x02, 0x1b, 0x83, 0x93, 0x53, 0x8c, 0x9b, 0x56, 0x6b, 0xfb, 0xe5, 0x75, 0xe7, 0x83, 0x55, 0x9c, + 0x7a, 0xc4, 0xe2, 0xd0, 0xc3, 0xec, 0x3c, 0x9b, 0xaa, 0xfe, 0xb6, 0x06, 0xb6, 0xa9, 0x8f, 0x77, + 0x3d, 0x78, 0x6f, 0x26, 0x2d, 0x27, 0x55, 0xfd, 0xda, 0x1b, 0x3b, 0x4f, 0x23, 0x21, 0x98, 0x3b, + 0x7e, 0x08, 0x0d, 0x3f, 0x9a, 0x7f, 0x3e, 0xa6, 0x21, 0x39, 0x0f, 0xcc, 0x80, 0x55, 0xc3, 0xb6, + 0xb2, 0x0d, 0x13, 0x93, 0xfa, 0xe4, 0xfc, 0x50, 0x52, 0x1e, 0x9a, 0xd1, 0xa9, 0x86, 0xb3, 0x3d, + 0xfa, 0x12, 0xca, 0x7e, 0x44, 0x66, 0xa6, 0x5d, 0x16, 0x9e, 0xe0, 0x70, 0xd4, 0x3b, 0x36, 0x1a, + 0xec, 0xd7, 0x16, 0x37, 0xdb, 0x65, 0x65, 0xc0, 0x9a, 0x86, 0xda, 0x69, 0xc7, 0x52, 0x6f, 0xd2, + 0x15, 0xb4, 0x86, 0x73, 0x96, 0xee, 0x7f, 0xcb, 0x60, 0xef, 0x07, 0xb1, 0x90, 0xa6, 0x0f, 0xdc, + 0x59, 0xde, 0x5e, 0xc0, 0x16, 0xd1, 0x33, 0x38, 0x09, 0x55, 0x51, 0xd5, 0x93, 0x80, 0xc9, 0xdd, + 0xa3, 0x42, 0x77, 0x19, 0x38, 0x99, 0x1a, 0xfa, 0x15, 0xe5, 0xd3, 0xb1, 0x70, 0x93, 0x7c, 0xe7, + 0x09, 0x3a, 0x85, 0x4d, 0xc6, 0xdd, 0x29, 0x15, 0x32, 0xa9, 0xc3, 0x66, 0x66, 0x2d, 0xfc, 0x9b, + 0xf9, 0x2a, 0x0f, 0x34, 0x45, 0x28, 0x89, 0x76, 0xd5, 0x07, 0x7a, 0x02, 0x65, 0x4e, 0x2e, 0xd2, + 0xa9, 0xa6, 0x50, 0xdf, 0x98, 0x5c, 0xc8, 0x15, 0x17, 0x9a, 0x81, 0x7e, 0x0f, 0xe0, 0xf9, 0x22, + 0x22, 0xd2, 0x9d, 0x52, 0x6e, 0xee, 0xa9, 0xf0, 0x88, 0x83, 0x0c, 0xb5, 0xe2, 0x25, 0xc7, 0x46, + 0x47, 0x50, 0x77, 0x49, 0xaa, 0xb4, 0xca, 0xed, 0x2d, 0x68, 0xbf, 0x67, 0x5c, 0x34, 0x95, 0x8b, + 0xc5, 0xcd, 0x76, 0x2d, 0xb5, 0xe0, 0x9a, 0x4b, 0x8c, 0xf2, 0x8e, 0x60, 0x53, 0x0d, 0xf8, 0x63, + 0x8f, 0x5e, 0x90, 0x38, 0x90, 0x42, 0x77, 0xcb, 0x5b, 0x8a, 0xaa, 0x9a, 0x35, 0x07, 0x06, 0x67, + 0xe2, 0x6a, 0xc8, 0x9c, 0x0d, 0xfd, 0x11, 0xb6, 0x68, 0xe8, 0xf2, 0x2b, 0xad, 0xb3, 0x34, 0xc2, + 0xda, 0xed, 0x87, 0x1d, 0x66, 0xe0, 0x95, 0xc3, 0x36, 0xe9, 0x77, 0xec, 0x5d, 0x1f, 0x20, 0x69, + 0x53, 0x77, 0xab, 0x3f, 0x04, 0x65, 0x8f, 0x48, 0xa2, 0x25, 0xd7, 0xc0, 0x7a, 0xdd, 0xff, 0xf0, + 0xd5, 0xeb, 0x76, 0xe9, 0x1f, 0xaf, 0xdb, 0xa5, 0x7f, 0xbf, 0x6e, 0x5b, 0x7f, 0x5e, 0xb4, 0xad, + 0x57, 0x8b, 0xb6, 0xf5, 0xf7, 0x45, 0xdb, 0xfa, 0xd7, 0xa2, 0x6d, 0x9d, 0x57, 0xf4, 0xff, 0xfb, + 0xcf, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x66, 0xb4, 0xcd, 0xfe, 0x1e, 0x10, 0x00, 0x00, } diff --git a/api/specs.proto b/api/specs.proto index c3be5f4b17..4b91395599 100644 --- a/api/specs.proto +++ b/api/specs.proto @@ -41,8 +41,8 @@ message NodeSpec { DRAIN = 2 [(gogoproto.enumvalue_customname) = "NodeAvailabilityDrain"]; } - // Role defines the role the node should have. - NodeRole role = 2; + // DesiredRole defines the role the node should have. + NodeRole desired_role = 2; // Membership controls the admission of the node into the cluster. Membership membership = 3; diff --git a/ca/server.go b/ca/server.go index fa55d38534..e1245ef185 100644 --- a/ca/server.go +++ b/ca/server.go @@ -250,7 +250,8 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod // Create a new node err := s.store.Update(func(tx store.Tx) error { node := &api.Node{ - ID: nodeID, + Role: role, + ID: nodeID, Certificate: api.Certificate{ CSR: request.CSR, CN: nodeID, @@ -260,7 +261,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod }, }, Spec: api.NodeSpec{ - Role: role, + DesiredRole: role, Membership: api.NodeMembershipAccepted, Availability: request.Availability, }, @@ -318,7 +319,7 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [ cert = api.Certificate{ CSR: csr, CN: node.ID, - Role: node.Spec.Role, + Role: node.Role, Status: api.IssuanceStatus{ State: api.IssuanceStateRenew, }, diff --git a/ca/testutils/cautils.go b/ca/testutils/cautils.go index c7ddea63f5..8474a11c0f 100644 --- a/ca/testutils/cautils.go +++ b/ca/testutils/cautils.go @@ -220,9 +220,10 @@ func createNode(s *store.MemoryStore, nodeID, role string, csr, cert []byte) err Certificate: cert, }, Spec: api.NodeSpec{ - Role: apiRole, - Membership: api.NodeMembershipAccepted, + DesiredRole: apiRole, + Membership: api.NodeMembershipAccepted, }, + Role: apiRole, } return store.CreateNode(tx, node) diff --git a/cmd/swarmctl/node/common.go b/cmd/swarmctl/node/common.go index aaf1142942..a53a3a6eb3 100644 --- a/cmd/swarmctl/node/common.go +++ b/cmd/swarmctl/node/common.go @@ -113,11 +113,11 @@ func changeNodeRole(cmd *cobra.Command, args []string, role api.NodeRole) error } spec := &node.Spec - if spec.Role == role { + if spec.DesiredRole == role { return errNoChange } - spec.Role = role + spec.DesiredRole = role _, err = c.UpdateNode(common.Context(cmd), &api.UpdateNodeRequest{ NodeID: node.ID, diff --git a/cmd/swarmctl/node/list.go b/cmd/swarmctl/node/list.go index ff95281cd2..3e577a8bab 100644 --- a/cmd/swarmctl/node/list.go +++ b/cmd/swarmctl/node/list.go @@ -61,7 +61,7 @@ var ( reachability = reachability + " *" } } - if reachability == "" && spec.Role == api.NodeRoleManager { + if reachability == "" && spec.DesiredRole == api.NodeRoleManager { reachability = "UNKNOWN" } diff --git a/integration/cluster.go b/integration/cluster.go index 5f414bbe21..4680f4e551 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -270,7 +270,7 @@ func (c *testCluster) SetNodeRole(id string, role api.NodeRole) error { return err } spec := resp.Node.Spec.Copy() - spec.Role = role + spec.DesiredRole = role if _, err := c.api.UpdateNode(context.Background(), &api.UpdateNodeRequest{ NodeID: id, Spec: spec, diff --git a/integration/integration_test.go b/integration/integration_test.go index 21cb625e3f..0a7ac3c750 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -60,18 +60,18 @@ func pollClusterReady(t *testing.T, c *testCluster, numWorker, numManager int) { var leaderFound bool for _, n := range res.Nodes { if n.Status.State != api.NodeStatus_READY { - return fmt.Errorf("node %s with role %s isn't ready, status %s, message %s", n.ID, n.Spec.Role, n.Status.State, n.Status.Message) + return fmt.Errorf("node %s with desired role %s isn't ready, status %s, message %s", n.ID, n.Spec.DesiredRole, n.Status.State, n.Status.Message) } if n.Spec.Membership != api.NodeMembershipAccepted { - return fmt.Errorf("node %s with role %s isn't accepted to cluster, membership %s", n.ID, n.Spec.Role, n.Spec.Membership) + return fmt.Errorf("node %s with desired role %s isn't accepted to cluster, membership %s", n.ID, n.Spec.DesiredRole, n.Spec.Membership) } - if n.Certificate.Role != n.Spec.Role { - return fmt.Errorf("node %s had different roles in spec and certificate, %s and %s respectively", n.ID, n.Spec.Role, n.Certificate.Role) + if n.Certificate.Role != n.Spec.DesiredRole { + return fmt.Errorf("node %s had different roles in spec and certificate, %s and %s respectively", n.ID, n.Spec.DesiredRole, n.Certificate.Role) } if n.Certificate.Status.State != api.IssuanceStateIssued { - return fmt.Errorf("node %s with role %s has no issued certificate, issuance state %s", n.ID, n.Spec.Role, n.Certificate.Status.State) + return fmt.Errorf("node %s with desired role %s has no issued certificate, issuance state %s", n.ID, n.Spec.DesiredRole, n.Certificate.Status.State) } - if n.Spec.Role == api.NodeRoleManager { + if n.Role == api.NodeRoleManager { if n.ManagerStatus == nil { return fmt.Errorf("manager node %s has no ManagerStatus field", n.ID) } @@ -339,7 +339,7 @@ func TestDemoteDownedManager(t *testing.T) { resp, err := cl.api.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodeID}) require.NoError(t, err) spec := resp.Node.Spec.Copy() - spec.Role = api.NodeRoleWorker + spec.DesiredRole = api.NodeRoleWorker // stop the node, then demote it, and start it back up again so when it comes back up it has to realize // it's not running anymore diff --git a/manager/constraint/constraint.go b/manager/constraint/constraint.go index ae636cb109..a690bd3e2d 100644 --- a/manager/constraint/constraint.go +++ b/manager/constraint/constraint.go @@ -122,7 +122,7 @@ func NodeMatches(constraints []Constraint, n *api.Node) bool { return false } case strings.EqualFold(constraint.key, "node.role"): - if !constraint.Match(n.Spec.Role.String()) { + if !constraint.Match(n.Role.String()) { return false } case strings.EqualFold(constraint.key, "node.platform.os"): diff --git a/manager/controlapi/node.go b/manager/controlapi/node.go index 6689e6fddc..d880ab981d 100644 --- a/manager/controlapi/node.go +++ b/manager/controlapi/node.go @@ -146,7 +146,7 @@ func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) ( return true } for _, c := range request.Filters.Roles { - if c == e.Spec.Role { + if c == e.Role { return true } } @@ -205,7 +205,6 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) var ( node *api.Node member *membership.Member - demote bool ) err := s.store.Update(func(tx store.Tx) error { @@ -215,9 +214,7 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) } // Demotion sanity checks. - if node.Spec.Role == api.NodeRoleManager && request.Spec.Role == api.NodeRoleWorker { - demote = true - + if node.Spec.DesiredRole == api.NodeRoleManager && request.Spec.DesiredRole == api.NodeRoleWorker { // Check for manager entries in Store. managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager)) if err != nil { @@ -246,16 +243,6 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) return nil, err } - if demote && s.raft != nil { - // TODO(abronan): the remove can potentially fail and leave the node with - // an incorrect role (worker rather than manager), we need to reconcile the - // memberlist with the desired state rather than attempting to remove the - // member once. - if err := s.raft.RemoveMember(ctx, member.RaftID); err != nil { - return nil, grpc.Errorf(codes.Internal, "cannot demote manager to worker: %v", err) - } - } - return &api.UpdateNodeResponse{ Node: node, }, nil @@ -276,7 +263,7 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) if node == nil { return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) } - if node.Spec.Role == api.NodeRoleManager { + if node.Spec.DesiredRole == api.NodeRoleManager { if s.raft == nil { return grpc.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) } diff --git a/manager/controlapi/node_test.go b/manager/controlapi/node_test.go index 01be370958..84cdaf9b03 100644 --- a/manager/controlapi/node_test.go +++ b/manager/controlapi/node_test.go @@ -24,12 +24,12 @@ func createNode(t *testing.T, ts *testServer, id string, role api.NodeRole, memb node := &api.Node{ ID: id, Spec: api.NodeSpec{ - Role: role, Membership: membership, }, Status: api.NodeStatus{ State: state, }, + Role: role, } err := ts.Store.Update(func(tx store.Tx) error { return store.CreateNode(tx, node) @@ -465,9 +465,9 @@ func TestUpdateNode(t *testing.T) { assert.NoError(t, store.CreateNode(tx, &api.Node{ ID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID(), Spec: api.NodeSpec{ - Role: api.NodeRoleManager, Membership: api.NodeMembershipAccepted, }, + Role: api.NodeRoleManager, })) return nil })) @@ -543,23 +543,26 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { assert.NoError(t, store.CreateNode(tx, &api.Node{ ID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID(), Spec: api.NodeSpec{ - Role: api.NodeRoleManager, - Membership: api.NodeMembershipAccepted, + DesiredRole: api.NodeRoleManager, + Membership: api.NodeMembershipAccepted, }, + Role: api.NodeRoleManager, })) assert.NoError(t, store.CreateNode(tx, &api.Node{ ID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID(), Spec: api.NodeSpec{ - Role: api.NodeRoleManager, - Membership: api.NodeMembershipAccepted, + DesiredRole: api.NodeRoleManager, + Membership: api.NodeMembershipAccepted, }, + Role: api.NodeRoleManager, })) assert.NoError(t, store.CreateNode(tx, &api.Node{ ID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID(), Spec: api.NodeSpec{ - Role: api.NodeRoleManager, - Membership: api.NodeMembershipAccepted, + DesiredRole: api.NodeRoleManager, + Membership: api.NodeMembershipAccepted, }, + Role: api.NodeRoleManager, })) return nil })) @@ -584,7 +587,7 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { r, err := ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID()}) assert.NoError(t, err) spec := r.Node.Spec.Copy() - spec.Role = api.NodeRoleWorker + spec.DesiredRole = api.NodeRoleWorker version := &r.Node.Meta.Version _, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{ NodeID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID(), @@ -610,11 +613,14 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { return nil })) + raftMember := ts.Server.raft.GetMemberByNodeID(nodes[3].SecurityConfig.ClientTLSCreds.NodeID()) + assert.NotNil(t, raftMember) + // Try to demote Node 3, this should succeed r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID()}) assert.NoError(t, err) spec = r.Node.Spec.Copy() - spec.Role = api.NodeRoleWorker + spec.DesiredRole = api.NodeRoleWorker version = &r.Node.Meta.Version _, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{ NodeID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID(), @@ -628,6 +634,8 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { 2: nodes[2], } + ts.Server.raft.RemoveMember(context.Background(), raftMember.RaftID) + raftutils.WaitForCluster(t, clockSource, newCluster) // Server should list 2 members @@ -648,11 +656,14 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { lastNode = nodes[1] } + raftMember = ts.Server.raft.GetMemberByNodeID(demoteNode.SecurityConfig.ClientTLSCreds.NodeID()) + assert.NotNil(t, raftMember) + // Try to demote a Node and scale down to 1 r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: demoteNode.SecurityConfig.ClientTLSCreds.NodeID()}) assert.NoError(t, err) spec = r.Node.Spec.Copy() - spec.Role = api.NodeRoleWorker + spec.DesiredRole = api.NodeRoleWorker version = &r.Node.Meta.Version _, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{ NodeID: demoteNode.SecurityConfig.ClientTLSCreds.NodeID(), @@ -661,6 +672,8 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { }) assert.NoError(t, err) + ts.Server.raft.RemoveMember(context.Background(), raftMember.RaftID) + // Update the server ts.Server.raft = lastNode.Node ts.Server.store = lastNode.MemoryStore() @@ -683,7 +696,7 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: lastNode.SecurityConfig.ClientTLSCreds.NodeID()}) assert.NoError(t, err) spec = r.Node.Spec.Copy() - spec.Role = api.NodeRoleWorker + spec.DesiredRole = api.NodeRoleWorker version = &r.Node.Meta.Version _, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{ NodeID: lastNode.SecurityConfig.ClientTLSCreds.NodeID(), @@ -710,7 +723,6 @@ func testUpdateNodeDemote(leader bool, t *testing.T) { r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: lastNode.SecurityConfig.ClientTLSCreds.NodeID()}) assert.NoError(t, err) assert.Equal(t, r.Node.Spec.Availability, api.NodeAvailabilityDrain) - } func TestUpdateNodeDemote(t *testing.T) { diff --git a/manager/manager.go b/manager/manager.go index c81bfbafea..2168a25495 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -125,6 +125,7 @@ type Manager struct { localserver *grpc.Server raftNode *raft.Node dekRotator *RaftDEKManager + roleManager *roleManager cancelFunc context.CancelFunc @@ -270,6 +271,12 @@ func New(config *Config) (*Manager, error) { return m, nil } +// RemovedFromRaft returns a channel that's closed if the manager is removed +// from the raft cluster. This should be used to trigger a manager shutdown. +func (m *Manager) RemovedFromRaft() <-chan struct{} { + return m.raftNode.RemovedFromRaft +} + // Addr returns tcp address on which remote api listens. func (m *Manager) Addr() string { return m.config.RemoteAPI.ListenAddr @@ -388,39 +395,26 @@ func (m *Manager) Run(parent context.Context) error { close(m.started) - errCh := make(chan error, 1) go func() { err := m.raftNode.Run(ctx) if err != nil { - errCh <- err log.G(ctx).WithError(err).Error("raft node stopped") - m.Stop(ctx) + m.Stop(ctx, false) } }() - returnErr := func(err error) error { - select { - case runErr := <-errCh: - if runErr == raft.ErrMemberRemoved { - return runErr - } - default: - } - return err - } - if err := raft.WaitForLeader(ctx, m.raftNode); err != nil { - return returnErr(err) + return err } c, err := raft.WaitForCluster(ctx, m.raftNode) if err != nil { - return returnErr(err) + return err } raftConfig := c.Spec.Raft if err := m.watchForKEKChanges(ctx); err != nil { - return returnErr(err) + return err } if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick { @@ -438,16 +432,17 @@ func (m *Manager) Run(parent context.Context) error { return nil } m.mu.Unlock() - m.Stop(ctx) + m.Stop(ctx, false) - return returnErr(err) + return err } const stopTimeout = 8 * time.Second // Stop stops the manager. It immediately closes all open connections and -// active RPCs as well as stopping the scheduler. -func (m *Manager) Stop(ctx context.Context) { +// active RPCs as well as stopping the scheduler. If clearData is set, the +// raft logs, snapshots, and keys will be erased. +func (m *Manager) Stop(ctx context.Context, clearData bool) { log.G(ctx).Info("Stopping manager") // It's not safe to start shutting down while the manager is still // starting up. @@ -472,6 +467,8 @@ func (m *Manager) Stop(ctx context.Context) { close(localSrvDone) }() + m.raftNode.Cancel() + m.dispatcher.Stop() m.logbroker.Stop() m.caserver.Stop() @@ -494,10 +491,16 @@ func (m *Manager) Stop(ctx context.Context) { if m.scheduler != nil { m.scheduler.Stop() } + if m.roleManager != nil { + m.roleManager.Stop() + } if m.keyManager != nil { m.keyManager.Stop() } + if clearData { + m.raftNode.ClearData() + } m.cancelFunc() <-m.raftNode.Done() @@ -778,6 +781,7 @@ func (m *Manager) becomeLeader(ctx context.Context) { m.taskReaper = taskreaper.New(s) m.scheduler = scheduler.New(s) m.keyManager = keymanager.New(s, keymanager.DefaultConfig()) + m.roleManager = newRoleManager(s, m.raftNode) // TODO(stevvooe): Allocate a context that can be used to // shutdown underlying manager processes when leadership is @@ -853,6 +857,9 @@ func (m *Manager) becomeLeader(ctx context.Context) { } }(m.globalOrchestrator) + go func(roleManager *roleManager) { + roleManager.Run() + }(m.roleManager) } // becomeFollower shuts down the subsystems that are only run by the leader. @@ -881,6 +888,9 @@ func (m *Manager) becomeFollower() { m.scheduler.Stop() m.scheduler = nil + m.roleManager.Stop() + m.roleManager = nil + if m.keyManager != nil { m.keyManager.Stop() m.keyManager = nil @@ -937,7 +947,7 @@ func managerNode(nodeID string, availability api.NodeSpec_Availability) *api.Nod }, }, Spec: api.NodeSpec{ - Role: api.NodeRoleManager, + DesiredRole: api.NodeRoleManager, Membership: api.NodeMembershipAccepted, Availability: availability, }, diff --git a/manager/manager_test.go b/manager/manager_test.go index e34cf716d6..e7211eb154 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -185,7 +185,7 @@ func TestManager(t *testing.T) { _, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{}) assert.Contains(t, grpc.ErrorDesc(err), "removed from swarm") - m.Stop(ctx) + m.Stop(ctx, false) // After stopping we should MAY receive an error from ListenAndServe if // all this happened before WaitForLeader completed, so don't check the @@ -393,7 +393,7 @@ func TestManagerLockUnlock(t *testing.T) { require.NotNil(t, unencryptedDEK) require.Equal(t, currentDEK, unencryptedDEK) - m.Stop(ctx) + m.Stop(ctx, false) // After stopping we should MAY receive an error from ListenAndServe if // all this happened before WaitForLeader completed, so don't check the diff --git a/manager/orchestrator/constraintenforcer/constraint_enforcer_test.go b/manager/orchestrator/constraintenforcer/constraint_enforcer_test.go index 7edd196c2e..526076fdac 100644 --- a/manager/orchestrator/constraintenforcer/constraint_enforcer_test.go +++ b/manager/orchestrator/constraintenforcer/constraint_enforcer_test.go @@ -18,12 +18,12 @@ func TestConstraintEnforcer(t *testing.T) { Annotations: api.Annotations{ Name: "name1", }, - Role: api.NodeRoleWorker, Availability: api.NodeAvailabilityActive, }, Status: api.NodeStatus{ State: api.NodeStatus_READY, }, + Role: api.NodeRoleWorker, }, { ID: "id2", @@ -141,7 +141,7 @@ func TestConstraintEnforcer(t *testing.T) { if node == nil { t.Fatal("could not get node id1") } - node.Spec.Role = api.NodeRoleManager + node.Role = api.NodeRoleManager assert.NoError(t, store.UpdateNode(tx, node)) return nil }) diff --git a/manager/role_manager.go b/manager/role_manager.go new file mode 100644 index 0000000000..63dbfb8505 --- /dev/null +++ b/manager/role_manager.go @@ -0,0 +1,167 @@ +package manager + +import ( + "time" + + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/state" + "github.com/docker/swarmkit/manager/state/raft" + "github.com/docker/swarmkit/manager/state/store" + "golang.org/x/net/context" +) + +const roleReconcileInterval = 5 * time.Second + +// roleManager reconciles the raft member list with desired role changes. +type roleManager struct { + ctx context.Context + cancel func() + + store *store.MemoryStore + raft *raft.Node + doneChan chan struct{} + + // pending contains changed nodes that have not yet been reconciled in + // the raft member list. + pending map[string]*api.Node +} + +// newRoleManager creates a new roleManager. +func newRoleManager(store *store.MemoryStore, raftNode *raft.Node) *roleManager { + ctx, cancel := context.WithCancel(context.Background()) + return &roleManager{ + ctx: ctx, + cancel: cancel, + store: store, + raft: raftNode, + doneChan: make(chan struct{}), + pending: make(map[string]*api.Node), + } +} + +// Run is roleManager's main loop. +func (rm *roleManager) Run() { + defer close(rm.doneChan) + + var ( + nodes []*api.Node + ticker *time.Ticker + tickerCh <-chan time.Time + ) + + watcher, cancelWatch, err := store.ViewAndWatch(rm.store, + func(readTx store.ReadTx) error { + var err error + nodes, err = store.FindNodes(readTx, store.All) + return err + }, + state.EventUpdateNode{}) + defer cancelWatch() + + if err != nil { + log.L.WithError(err).Error("failed to check nodes for role changes") + } else { + for _, node := range nodes { + rm.pending[node.ID] = node + rm.reconcileRole(node) + } + if len(rm.pending) != 0 { + ticker = time.NewTicker(roleReconcileInterval) + tickerCh = ticker.C + } + } + + for { + select { + case event := <-watcher: + node := event.(state.EventUpdateNode).Node + rm.pending[node.ID] = node + rm.reconcileRole(node) + if len(rm.pending) != 0 && ticker == nil { + ticker = time.NewTicker(roleReconcileInterval) + tickerCh = ticker.C + } + case <-tickerCh: + for _, node := range rm.pending { + rm.reconcileRole(node) + } + if len(rm.pending) == 0 { + ticker.Stop() + ticker = nil + tickerCh = nil + } + case <-rm.ctx.Done(): + if ticker != nil { + ticker.Stop() + } + return + } + } +} + +func (rm *roleManager) reconcileRole(node *api.Node) { + if node.Role == node.Spec.DesiredRole { + // Nothing to do. + delete(rm.pending, node.ID) + return + } + + // Promotion can proceed right away. + if node.Spec.DesiredRole == api.NodeRoleManager && node.Role == api.NodeRoleWorker { + err := rm.store.Update(func(tx store.Tx) error { + updatedNode := store.GetNode(tx, node.ID) + if updatedNode == nil || updatedNode.Spec.DesiredRole != node.Spec.DesiredRole || updatedNode.Role != node.Role { + return nil + } + updatedNode.Role = api.NodeRoleManager + return store.UpdateNode(tx, updatedNode) + }) + if err != nil { + log.L.WithError(err).Errorf("failed to promote node %s", node.ID) + } else { + delete(rm.pending, node.ID) + } + } else if node.Spec.DesiredRole == api.NodeRoleWorker && node.Role == api.NodeRoleManager { + // Check for node in memberlist + member := rm.raft.GetMemberByNodeID(node.ID) + if member != nil { + // Quorum safeguard + if !rm.raft.CanRemoveMember(member.RaftID) { + // TODO(aaronl): Retry later + log.L.Debugf("can't demote node %s at this time: removing member from raft would result in a loss of quorum", node.ID) + return + } + + rmCtx, rmCancel := context.WithTimeout(rm.ctx, 5*time.Second) + defer rmCancel() + + if err := rm.raft.RemoveMember(rmCtx, member.RaftID); err != nil { + // TODO(aaronl): Retry later + log.L.WithError(err).Debugf("can't demote node %s at this time", node.ID) + return + } + } + + err := rm.store.Update(func(tx store.Tx) error { + updatedNode := store.GetNode(tx, node.ID) + if updatedNode == nil || updatedNode.Spec.DesiredRole != node.Spec.DesiredRole || updatedNode.Role != node.Role { + return nil + } + updatedNode.Role = api.NodeRoleWorker + + return store.UpdateNode(tx, updatedNode) + }) + if err != nil { + log.L.WithError(err).Errorf("failed to demote node %s", node.ID) + } else { + delete(rm.pending, node.ID) + } + } +} + +// Stop stops the roleManager and waits for the main loop to exit. +func (rm *roleManager) Stop() { + rm.cancel() + <-rm.doneChan +} diff --git a/manager/scheduler/constraint_test.go b/manager/scheduler/constraint_test.go index 168f6cf953..1be4cced06 100644 --- a/manager/scheduler/constraint_test.go +++ b/manager/scheduler/constraint_test.go @@ -42,7 +42,7 @@ func setupEnv() { Annotations: api.Annotations{ Labels: make(map[string]string), }, - Role: api.NodeRoleWorker, + DesiredRole: api.NodeRoleWorker, }, Description: &api.NodeDescription{ Engine: &api.EngineDescription{ diff --git a/manager/state/raft/raft.go b/manager/state/raft/raft.go index babe1a0c7e..c46727d937 100644 --- a/manager/state/raft/raft.go +++ b/manager/state/raft/raft.go @@ -112,9 +112,10 @@ type Node struct { ticker clock.Ticker doneCh chan struct{} - // removeRaftCh notifies about node deletion from raft cluster - removeRaftCh chan struct{} + // RemovedFromRaft notifies about node deletion from raft cluster + RemovedFromRaft chan struct{} removeRaftFunc func() + cancelFunc func() leadershipBroadcast *watch.Queue // used to coordinate shutdown @@ -134,6 +135,7 @@ type Node struct { raftLogger *storage.EncryptedRaftLogger keyRotator EncryptionKeyRotator rotationQueued bool + clearData bool waitForAppliedIndex uint64 } @@ -199,7 +201,7 @@ func NewNode(opts NodeOptions) *Node { Logger: cfg.Logger, }, doneCh: make(chan struct{}), - removeRaftCh: make(chan struct{}), + RemovedFromRaft: make(chan struct{}), stopped: make(chan struct{}), leadershipBroadcast: watch.NewQueue(), lastSendToMember: make(map[uint64]chan struct{}), @@ -220,7 +222,17 @@ func NewNode(opts NodeOptions) *Node { var removeRaftOnce sync.Once return func() { removeRaftOnce.Do(func() { - close(n.removeRaftCh) + atomic.StoreUint32(&n.isMember, 0) + close(n.RemovedFromRaft) + }) + } + }(n) + + n.cancelFunc = func(n *Node) func() { + var cancelOnce sync.Once + return func() { + cancelOnce.Do(func() { + close(n.stopped) }) } }(n) @@ -364,6 +376,12 @@ func (n *Node) done() { close(n.doneCh) } +// ClearData tells the raft node to delete its WALs, snapshots, and keys on +// shutdown. +func (n *Node) ClearData() { + n.clearData = true +} + // Run is the main loop for a Raft node, it goes along the state machine, // acting on the messages received from other Raft nodes in the cluster. // @@ -373,13 +391,10 @@ func (n *Node) Run(ctx context.Context) error { ctx = log.WithLogger(ctx, logrus.WithField("raft_id", fmt.Sprintf("%x", n.Config.ID))) ctx, cancel := context.WithCancel(ctx) - // nodeRemoved indicates that node was stopped due its removal. - nodeRemoved := false - defer func() { cancel() n.stop(ctx) - if nodeRemoved { + if n.clearData { // Delete WAL and snapshots, since they are no longer // usable. if err := n.raftLogger.Clear(ctx); err != nil { @@ -534,12 +549,6 @@ func (n *Node) Run(ctx context.Context) error { case n.needsSnapshot(ctx): n.doSnapshot(ctx, n.getCurrentRaftConfig()) } - case <-n.removeRaftCh: - nodeRemoved = true - // If the node was removed from other members, - // send back an error to the caller to start - // the shutdown process. - return ErrMemberRemoved case <-ctx.Done(): return nil } @@ -603,6 +612,15 @@ func (n *Node) getCurrentRaftConfig() api.RaftConfig { return raftConfig } +// Cancel interrupts all ongoing proposals, and prevents new ones from +// starting. This is useful for the shutdown sequence because it allows +// the manager to shut down raft-dependent services that might otherwise +// block on shutdown if quorum isn't met. Then the raft node can be completely +// shut down once no more code is using it. +func (n *Node) Cancel() { + n.cancelFunc() +} + // Done returns channel which is closed when raft node is fully stopped. func (n *Node) Done() <-chan struct{} { return n.doneCh @@ -612,8 +630,7 @@ func (n *Node) stop(ctx context.Context) { n.stopMu.Lock() defer n.stopMu.Unlock() - close(n.stopped) - + n.Cancel() n.waitProp.Wait() n.asyncTasks.Wait() @@ -1238,17 +1255,6 @@ func (n *Node) IsMember() bool { return atomic.LoadUint32(&n.isMember) == 1 } -// canSubmitProposal defines if any more proposals -// could be submitted and processed. -func (n *Node) canSubmitProposal() bool { - select { - case <-n.stopped: - return false - default: - return true - } -} - // Saves a log entry to our Store func (n *Node) saveToStorage( ctx context.Context, @@ -1472,7 +1478,7 @@ func (n *Node) handleAddressChange(ctx context.Context, member *membership.Membe // shutdown. func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRaftRequest, cb func()) (proto.Message, error) { n.stopMu.RLock() - if !n.canSubmitProposal() { + if !n.IsMember() { n.stopMu.RUnlock() return nil, ErrStopped } @@ -1718,11 +1724,11 @@ func (n *Node) applyRemoveNode(ctx context.Context, cc raftpb.ConfChange) (err e } if cc.NodeID == n.Config.ID { - n.removeRaftFunc() - // wait the commit ack to be sent before closing connection n.asyncTasks.Wait() + n.removeRaftFunc() + // if there are only 2 nodes in the cluster, and leader is leaving // before closing the connection, leader has to ensure that follower gets // noticed about this raft conf change commit. Otherwise, follower would diff --git a/manager/state/raft/raft_test.go b/manager/state/raft/raft_test.go index bdd4a8b93f..634034021a 100644 --- a/manager/state/raft/raft_test.go +++ b/manager/state/raft/raft_test.go @@ -7,7 +7,6 @@ import ( "log" "math/rand" "os" - "path/filepath" "reflect" "strconv" "testing" @@ -20,7 +19,6 @@ import ( "golang.org/x/net/context" "github.com/Sirupsen/logrus" - "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/wal" "github.com/docker/swarmkit/api" cautils "github.com/docker/swarmkit/ca/testutils" @@ -341,11 +339,6 @@ func TestRaftLeaderLeave(t *testing.T) { // Wait for election tick raftutils.WaitForCluster(t, clockSource, newCluster) - // Node1's state should be cleared - require.False(t, fileutil.Exist(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted"))) - require.False(t, fileutil.Exist(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted"))) - require.Equal(t, raft.EncryptionKeys{}, nodes[1].KeyRotator.GetKeys()) - // Leader should not be 1 assert.NotEqual(t, nodes[2].Leader(), nodes[1].Config.ID) assert.Equal(t, nodes[2].Leader(), nodes[3].Leader()) diff --git a/manager/state/store/memory_test.go b/manager/state/store/memory_test.go index f3193cb9d3..431c3cc751 100644 --- a/manager/state/store/memory_test.go +++ b/manager/state/store/memory_test.go @@ -19,33 +19,33 @@ var ( { ID: "id1", Spec: api.NodeSpec{ - Role: api.NodeRoleManager, Membership: api.NodeMembershipPending, }, Description: &api.NodeDescription{ Hostname: "name1", }, + Role: api.NodeRoleManager, }, { ID: "id2", Spec: api.NodeSpec{ - Role: api.NodeRoleWorker, Membership: api.NodeMembershipAccepted, }, Description: &api.NodeDescription{ Hostname: "name2", }, + Role: api.NodeRoleWorker, }, { ID: "id3", Spec: api.NodeSpec{ - Role: api.NodeRoleWorker, Membership: api.NodeMembershipAccepted, }, Description: &api.NodeDescription{ // intentionally conflicting hostname Hostname: "name2", }, + Role: api.NodeRoleWorker, }, } diff --git a/manager/state/store/nodes.go b/manager/state/store/nodes.go index 04985fc5a2..ec4709abde 100644 --- a/manager/state/store/nodes.go +++ b/manager/state/store/nodes.go @@ -238,7 +238,7 @@ func (ni nodeIndexerByRole) FromObject(obj interface{}) (bool, []byte, error) { } // Add the null character as a terminator - return true, []byte(strconv.FormatInt(int64(n.Spec.Role), 10) + "\x00"), nil + return true, []byte(strconv.FormatInt(int64(n.Role), 10) + "\x00"), nil } type nodeIndexerByMembership struct{} diff --git a/node/node.go b/node/node.go index 57b88a9f93..cf1545c529 100644 --- a/node/node.go +++ b/node/node.go @@ -22,7 +22,6 @@ import ( "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager" "github.com/docker/swarmkit/manager/encryption" - "github.com/docker/swarmkit/manager/state/raft" "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/xnet" "github.com/pkg/errors" @@ -249,18 +248,13 @@ func (n *Node) run(ctx context.Context) (err error) { // If we got a role change, renew lastRole := n.role role := ca.WorkerRole - if node.Spec.Role == api.NodeRoleManager { + if node.Role == api.NodeRoleManager { role = ca.ManagerRole } if lastRole == role { n.Unlock() continue } - // switch role to agent immediately to shutdown manager early - if role == ca.WorkerRole { - n.role = role - n.roleCond.Broadcast() - } n.Unlock() renewCert() } @@ -308,7 +302,18 @@ func (n *Node) run(ctx context.Context) (err error) { go func() { <-agentReady if role == ca.ManagerRole { - <-managerReady + workerRole := make(chan struct{}) + waitRoleCtx, waitRoleCancel := context.WithCancel(ctx) + go func() { + if n.waitRole(waitRoleCtx, ca.WorkerRole) == nil { + close(workerRole) + } + }() + select { + case <-managerReady: + case <-workerRole: + } + waitRoleCancel() } close(n.ready) }() @@ -632,7 +637,7 @@ func (n *Node) waitRole(ctx context.Context, role string) error { return nil } -func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig, ready chan struct{}) error { +func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig, ready chan struct{}, workerRole <-chan struct{}) error { remoteAddr, _ := n.remotes.Select(n.NodeID()) m, err := manager.New(&manager.Config{ ForceNewCluster: n.config.ForceNewCluster, @@ -657,25 +662,18 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig done := make(chan struct{}) var runErr error go func() { - if err := m.Run(context.Background()); err != nil && err != raft.ErrMemberRemoved { + if err := m.Run(context.Background()); err != nil { runErr = err } close(done) }() - workerRole := make(chan struct{}) - waitRoleCtx, waitRoleCancel := context.WithCancel(ctx) - defer waitRoleCancel() - go func() { - n.waitRole(waitRoleCtx, ca.WorkerRole) - close(workerRole) - }() - + var clearData bool defer func() { n.Lock() n.manager = nil n.Unlock() - m.Stop(ctx) + m.Stop(ctx, clearData) <-done n.setControlSocket(nil) }() @@ -706,33 +704,19 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig } // wait for manager stop or for role change - // if manager stopped before role change, wait for new role for 16 seconds, - // then just restart manager, we might just miss that event. - // we need to wait for role to prevent manager to start again with wrong - // certificate select { case <-done: - timer := time.NewTimer(16 * time.Second) - defer timer.Stop() - select { - case <-timer.C: - log.G(ctx).Warn("failed to get worker role after manager stop, restart manager") - case <-workerRole: - case <-ctx.Done(): - return ctx.Err() - } return runErr case <-workerRole: - log.G(ctx).Info("role changed to worker, wait for manager to stop") - select { - case <-done: - return runErr - case <-ctx.Done(): - return ctx.Err() - } + log.G(ctx).Info("role changed to worker, stopping manager") + clearData = true + case <-m.RemovedFromRaft(): + log.G(ctx).Info("manager removed from raft cluster, stopping manager") + clearData = true case <-ctx.Done(): return ctx.Err() } + return nil } func (n *Node) superviseManager(ctx context.Context, securityConfig *ca.SecurityConfig, ready chan struct{}) error { @@ -740,9 +724,37 @@ func (n *Node) superviseManager(ctx context.Context, securityConfig *ca.Security if err := n.waitRole(ctx, ca.ManagerRole); err != nil { return err } - if err := n.runManager(ctx, securityConfig, ready); err != nil { + + workerRole := make(chan struct{}) + waitRoleCtx, waitRoleCancel := context.WithCancel(ctx) + go func() { + if n.waitRole(waitRoleCtx, ca.WorkerRole) == nil { + close(workerRole) + } + }() + + if err := n.runManager(ctx, securityConfig, ready, workerRole); err != nil { + waitRoleCancel() return errors.Wrap(err, "manager stopped") } + + // If the manager stopped running and our role is still + // "manager", it's possible that the manager was demoted and + // the agent hasn't realized this yet. We should wait for the + // role to change instead of restarting the manager immediately. + timer := time.NewTimer(16 * time.Second) + select { + case <-timer.C: + log.G(ctx).Warn("failed to get worker role after manager stop, restarting manager") + case <-workerRole: + case <-ctx.Done(): + timer.Stop() + waitRoleCancel() + return ctx.Err() + } + timer.Stop() + waitRoleCancel() + ready = nil } }