From 33c050860e77e3968e8c7177ea6f11b12b6fdb60 Mon Sep 17 00:00:00 2001 From: Nikhil Benesch Date: Thu, 12 Jul 2018 23:41:58 -0400 Subject: [PATCH] storage: avoid deadlock during merges Merges could deadlock if the lease on the right-hand range was transferred or renewed during the merge transaction and the new leaseholder received any request before the GetSnapshotForMerge request. See #27442 and the comments within for details. Avoid the deadlock by moving the responsibility for waiting for a merge to complete from the replica to the store. Add a unit test that runs several get requests concurrently with several merges to reliably test for regressions. The problem previously only presented when stressing TestStoreRangeMergeWithData. Fix #27442. Release note: None --- pkg/roachpb/api.pb.go | 1 + pkg/roachpb/errors.go | 10 + pkg/roachpb/errors.pb.go | 501 +++++++++++++++++++++---------- pkg/roachpb/errors.proto | 10 + pkg/storage/client_merge_test.go | 114 ++++++- pkg/storage/replica.go | 31 +- pkg/storage/store.go | 18 ++ 7 files changed, 505 insertions(+), 180 deletions(-) diff --git a/pkg/roachpb/api.pb.go b/pkg/roachpb/api.pb.go index c388732fb468..9293823dec30 100644 --- a/pkg/roachpb/api.pb.go +++ b/pkg/roachpb/api.pb.go @@ -153,6 +153,7 @@ MixedSuccessError BatchTimestampBeforeGCError IntentMissingError + MergeInProgressError ErrorDetail ErrPosition Error diff --git a/pkg/roachpb/errors.go b/pkg/roachpb/errors.go index 19578809ac3f..b830e6499ddc 100644 --- a/pkg/roachpb/errors.go +++ b/pkg/roachpb/errors.go @@ -706,3 +706,13 @@ func (e *IntentMissingError) message(_ *Error) string { } var _ ErrorDetailInterface = &IntentMissingError{} + +func (e *MergeInProgressError) Error() string { + return e.message(nil) +} + +func (e *MergeInProgressError) message(_ *Error) string { + return "merge in progress" +} + +var _ ErrorDetailInterface = &MergeInProgressError{} diff --git a/pkg/roachpb/errors.pb.go b/pkg/roachpb/errors.pb.go index 1dc7a0fc3e83..ae39f7358c48 100644 --- a/pkg/roachpb/errors.pb.go +++ b/pkg/roachpb/errors.pb.go @@ -595,6 +595,19 @@ func (m *IntentMissingError) String() string { return proto.CompactTe func (*IntentMissingError) ProtoMessage() {} func (*IntentMissingError) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{29} } +// A MergeInProgressError indicates that the request could not be completed +// because the replica is being merged into its left-hand neighbor. The request +// should be resubmitted after the merge completes. +// +// This error is handled by the Store and should not escape to higher levels. +type MergeInProgressError struct { +} + +func (m *MergeInProgressError) Reset() { *m = MergeInProgressError{} } +func (m *MergeInProgressError) String() string { return proto.CompactTextString(m) } +func (*MergeInProgressError) ProtoMessage() {} +func (*MergeInProgressError) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{30} } + // ErrorDetail is a union type containing all available errors. type ErrorDetail struct { NotLeaseHolder *NotLeaseHolderError `protobuf:"bytes,1,opt,name=not_lease_holder,json=notLeaseHolder" json:"not_lease_holder,omitempty"` @@ -631,12 +644,13 @@ type ErrorDetail struct { TimestampBefore *BatchTimestampBeforeGCError `protobuf:"bytes,34,opt,name=timestamp_before,json=timestampBefore" json:"timestamp_before,omitempty"` TxnAlreadyEncounteredError *TxnAlreadyEncounteredErrorError `protobuf:"bytes,35,opt,name=txn_already_encountered_error,json=txnAlreadyEncounteredError" json:"txn_already_encountered_error,omitempty"` IntentMissing *IntentMissingError `protobuf:"bytes,36,opt,name=intent_missing,json=intentMissing" json:"intent_missing,omitempty"` + MergeInProgress *MergeInProgressError `protobuf:"bytes,37,opt,name=merge_in_progress,json=mergeInProgress" json:"merge_in_progress,omitempty"` } func (m *ErrorDetail) Reset() { *m = ErrorDetail{} } func (m *ErrorDetail) String() string { return proto.CompactTextString(m) } func (*ErrorDetail) ProtoMessage() {} -func (*ErrorDetail) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{30} } +func (*ErrorDetail) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{31} } // ErrPosition describes the position of an error in a Batch. A simple nullable // primitive field would break compatibility with proto3, where primitive fields @@ -648,7 +662,7 @@ type ErrPosition struct { func (m *ErrPosition) Reset() { *m = ErrPosition{} } func (m *ErrPosition) String() string { return proto.CompactTextString(m) } func (*ErrPosition) ProtoMessage() {} -func (*ErrPosition) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{31} } +func (*ErrPosition) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{32} } // Error is a generic representation including a string message // and information about retryability. @@ -678,7 +692,7 @@ type Error struct { func (m *Error) Reset() { *m = Error{} } func (*Error) ProtoMessage() {} -func (*Error) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{32} } +func (*Error) Descriptor() ([]byte, []int) { return fileDescriptorErrors, []int{33} } func init() { proto.RegisterType((*NotLeaseHolderError)(nil), "cockroach.roachpb.NotLeaseHolderError") @@ -711,6 +725,7 @@ func init() { proto.RegisterType((*MixedSuccessError)(nil), "cockroach.roachpb.MixedSuccessError") proto.RegisterType((*BatchTimestampBeforeGCError)(nil), "cockroach.roachpb.BatchTimestampBeforeGCError") proto.RegisterType((*IntentMissingError)(nil), "cockroach.roachpb.IntentMissingError") + proto.RegisterType((*MergeInProgressError)(nil), "cockroach.roachpb.MergeInProgressError") proto.RegisterType((*ErrorDetail)(nil), "cockroach.roachpb.ErrorDetail") proto.RegisterType((*ErrPosition)(nil), "cockroach.roachpb.ErrPosition") proto.RegisterType((*Error)(nil), "cockroach.roachpb.Error") @@ -1463,6 +1478,27 @@ func (this *IntentMissingError) Equal(that interface{}) bool { } return true } +func (this *MergeInProgressError) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*MergeInProgressError) + if !ok { + that2, ok := that.(MergeInProgressError) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + return true +} func (this *ErrorDetail) Equal(that interface{}) bool { if that == nil { return this == nil @@ -1569,6 +1605,9 @@ func (this *ErrorDetail) Equal(that interface{}) bool { if !this.IntentMissing.Equal(that1.IntentMissing) { return false } + if !this.MergeInProgress.Equal(that1.MergeInProgress) { + return false + } return true } func (this *ErrPosition) Equal(that interface{}) bool { @@ -2473,6 +2512,24 @@ func (m *IntentMissingError) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *MergeInProgressError) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MergeInProgressError) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func (m *ErrorDetail) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2806,6 +2863,18 @@ func (m *ErrorDetail) MarshalTo(dAtA []byte) (int, error) { } i += n51 } + if m.MergeInProgress != nil { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintErrors(dAtA, i, uint64(m.MergeInProgress.Size())) + n52, err := m.MergeInProgress.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n52 + } return i, nil } @@ -2856,11 +2925,11 @@ func (m *Error) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintErrors(dAtA, i, uint64(m.UnexposedTxn.Size())) - n52, err := m.UnexposedTxn.MarshalTo(dAtA[i:]) + n53, err := m.UnexposedTxn.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n52 + i += n53 } dAtA[i] = 0x28 i++ @@ -2869,30 +2938,30 @@ func (m *Error) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x32 i++ i = encodeVarintErrors(dAtA, i, uint64(m.Detail.Size())) - n53, err := m.Detail.MarshalTo(dAtA[i:]) + n54, err := m.Detail.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n53 + i += n54 } if m.Index != nil { dAtA[i] = 0x3a i++ i = encodeVarintErrors(dAtA, i, uint64(m.Index.Size())) - n54, err := m.Index.MarshalTo(dAtA[i:]) + n55, err := m.Index.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n54 + i += n55 } dAtA[i] = 0x42 i++ i = encodeVarintErrors(dAtA, i, uint64(m.Now.Size())) - n55, err := m.Now.MarshalTo(dAtA[i:]) + n56, err := m.Now.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n55 + i += n56 return i, nil } @@ -3194,6 +3263,12 @@ func (m *IntentMissingError) Size() (n int) { return n } +func (m *MergeInProgressError) Size() (n int) { + var l int + _ = l + return n +} + func (m *ErrorDetail) Size() (n int) { var l int _ = l @@ -3313,6 +3388,10 @@ func (m *ErrorDetail) Size() (n int) { l = m.IntentMissing.Size() n += 2 + l + sovErrors(uint64(l)) } + if m.MergeInProgress != nil { + l = m.MergeInProgress.Size() + n += 2 + l + sovErrors(uint64(l)) + } return n } @@ -3448,6 +3527,9 @@ func (this *ErrorDetail) GetValue() interface{} { if this.IntentMissing != nil { return this.IntentMissing } + if this.MergeInProgress != nil { + return this.MergeInProgress + } return nil } @@ -3511,6 +3593,8 @@ func (this *ErrorDetail) SetValue(value interface{}) bool { this.TxnAlreadyEncounteredError = vt case *IntentMissingError: this.IntentMissing = vt + case *MergeInProgressError: + this.MergeInProgress = vt default: return false } @@ -6233,6 +6317,56 @@ func (m *IntentMissingError) Unmarshal(dAtA []byte) error { } return nil } +func (m *MergeInProgressError) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowErrors + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MergeInProgressError: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MergeInProgressError: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipErrors(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthErrors + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ErrorDetail) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7219,6 +7353,39 @@ func (m *ErrorDetail) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 37: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MergeInProgress", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowErrors + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthErrors + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.MergeInProgress == nil { + m.MergeInProgress = &MergeInProgressError{} + } + if err := m.MergeInProgress.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipErrors(dAtA[iNdEx:]) @@ -7663,159 +7830,161 @@ var ( func init() { proto.RegisterFile("roachpb/errors.proto", fileDescriptorErrors) } var fileDescriptorErrors = []byte{ - // 2457 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcf, 0x53, 0x1b, 0xc9, - 0xf5, 0x47, 0x48, 0x20, 0x78, 0x42, 0x48, 0xb4, 0x31, 0x3b, 0xc6, 0x8b, 0xc4, 0xe2, 0xfd, 0x7e, - 0xd7, 0x76, 0x2a, 0xb0, 0x85, 0xe3, 0x4d, 0xad, 0xb3, 0x39, 0x08, 0x24, 0x58, 0x19, 0x90, 0x48, - 0x23, 0xd6, 0x66, 0x37, 0x55, 0x53, 0xc3, 0x4c, 0x23, 0x66, 0x2d, 0xcd, 0x28, 0x3d, 0x3d, 0x20, - 0x2e, 0x39, 0xe7, 0x98, 0xdc, 0x72, 0x8b, 0xab, 0x72, 0x4a, 0xe5, 0x92, 0x43, 0x2a, 0x97, 0x54, - 0xe5, 0xec, 0x63, 0x8e, 0xa9, 0x54, 0x85, 0x4a, 0xc8, 0x21, 0xa9, 0xca, 0x7f, 0xe0, 0x53, 0xaa, - 0x7f, 0x8c, 0x34, 0x42, 0x33, 0xb2, 0xe2, 0x93, 0x46, 0xaf, 0xdf, 0xfb, 0xf4, 0xeb, 0x1f, 0xef, - 0xbd, 0xcf, 0x6b, 0x58, 0xa4, 0xae, 0x61, 0x9e, 0x77, 0x4e, 0x37, 0x08, 0xa5, 0x2e, 0xf5, 0xd6, - 0x3b, 0xd4, 0x65, 0x2e, 0x5a, 0x30, 0x5d, 0xf3, 0x95, 0x18, 0x59, 0x57, 0xe3, 0xcb, 0x28, 0x50, - 0xb4, 0x0c, 0x66, 0x48, 0xb5, 0xe5, 0xa5, 0x40, 0xd6, 0x26, 0xcc, 0x08, 0xc9, 0x1f, 0x78, 0xcc, - 0xa5, 0x46, 0x93, 0x6c, 0x10, 0xa7, 0x69, 0x3b, 0xc1, 0x0f, 0xd7, 0xbb, 0x30, 0xcd, 0x27, 0x4a, - 0x49, 0xf3, 0x99, 0xdd, 0xda, 0x38, 0x6f, 0x99, 0x1b, 0xcc, 0x6e, 0x13, 0x8f, 0x19, 0xed, 0x8e, - 0x1a, 0x59, 0x6c, 0xba, 0x4d, 0x57, 0x7c, 0x6e, 0xf0, 0x2f, 0x29, 0x5d, 0xfb, 0xc3, 0x24, 0xdc, - 0xa9, 0xb9, 0x6c, 0x9f, 0x18, 0x1e, 0xf9, 0xd2, 0x6d, 0x59, 0x84, 0x56, 0xb8, 0xcb, 0xa8, 0x0c, - 0x69, 0x4a, 0x3a, 0x2d, 0xdb, 0x34, 0xb4, 0xc4, 0x6a, 0xe2, 0x61, 0x66, 0xf3, 0xe3, 0xf5, 0x21, - 0xef, 0xd7, 0xb1, 0xd4, 0x28, 0x13, 0xcf, 0xa4, 0x76, 0x87, 0xb9, 0x74, 0x2b, 0xf5, 0xe6, 0xba, - 0x38, 0x81, 0x03, 0x53, 0xb4, 0x0b, 0x73, 0x2d, 0x8e, 0xac, 0x9f, 0x0b, 0x68, 0x6d, 0x72, 0x7c, - 0x28, 0x9c, 0x69, 0xf5, 0x7d, 0x42, 0x4f, 0x61, 0x86, 0x1a, 0x4e, 0x93, 0xe8, 0xb6, 0xa5, 0x25, - 0x57, 0x13, 0x0f, 0x93, 0x5b, 0xcb, 0x7c, 0xa6, 0x9b, 0xeb, 0x62, 0x1a, 0x73, 0x79, 0xb5, 0xfc, - 0xb6, 0xff, 0x89, 0xd3, 0x42, 0xb7, 0x6a, 0xa1, 0x75, 0x98, 0x12, 0x28, 0x5a, 0x4a, 0x4c, 0xac, - 0x45, 0x4c, 0x2c, 0x56, 0x8e, 0xa5, 0x1a, 0x7a, 0x00, 0x60, 0xfa, 0x1e, 0x73, 0xdb, 0x7a, 0xdb, - 0x6b, 0x6a, 0x53, 0xab, 0x89, 0x87, 0xb3, 0x6a, 0x49, 0xb3, 0x52, 0x7e, 0xe0, 0x35, 0x9f, 0xa5, - 0xfe, 0xfd, 0xba, 0x98, 0x58, 0xfb, 0x10, 0x16, 0x6b, 0xae, 0x45, 0x8e, 0x1d, 0xe3, 0xc2, 0xb0, - 0x5b, 0xc6, 0x69, 0x8b, 0x88, 0x8d, 0x53, 0xa3, 0x45, 0xf8, 0xe0, 0xd8, 0xf1, 0xfc, 0x4e, 0xc7, - 0xa5, 0x8c, 0x58, 0x98, 0xfc, 0xc4, 0x27, 0x1e, 0x0b, 0x2b, 0xfc, 0x08, 0x90, 0xf0, 0xb6, 0xe6, - 0xb2, 0x1d, 0xd7, 0x77, 0x2c, 0xb9, 0xeb, 0xe1, 0x65, 0x26, 0xc6, 0x5e, 0xa6, 0x82, 0xfc, 0xd5, - 0x24, 0xdc, 0x15, 0x43, 0x7b, 0xe4, 0xea, 0xc0, 0xf6, 0xda, 0x06, 0x33, 0xcf, 0x25, 0xec, 0x13, - 0x58, 0xa0, 0xd2, 0x05, 0xdd, 0x63, 0x06, 0x65, 0xfa, 0x2b, 0x72, 0x25, 0xf0, 0xe7, 0xb6, 0xd2, - 0x6f, 0xaf, 0x8b, 0xc9, 0x3d, 0x72, 0x85, 0x73, 0x4a, 0xe3, 0x88, 0x2b, 0xec, 0x91, 0x2b, 0xb4, - 0x01, 0x81, 0x48, 0x27, 0x8e, 0x25, 0x4c, 0x26, 0x07, 0x4d, 0xb2, 0x6a, 0xbc, 0xe2, 0x58, 0xdc, - 0xe0, 0x00, 0xf2, 0x6d, 0x35, 0x2d, 0xb1, 0x74, 0xe1, 0x9b, 0x38, 0xab, 0xcc, 0xe6, 0x5a, 0xd4, - 0x81, 0xf3, 0xf1, 0xd0, 0x71, 0xe7, 0xfa, 0xb6, 0x62, 0x08, 0xed, 0x41, 0xce, 0xf3, 0x9b, 0x4d, - 0xe2, 0xb1, 0x1e, 0x5a, 0x6a, 0x6c, 0xb4, 0xf9, 0x9e, 0xa9, 0x18, 0x51, 0x3b, 0xf4, 0x9f, 0x49, - 0x58, 0xc3, 0xc4, 0xb0, 0x5e, 0xd8, 0xec, 0xdc, 0x76, 0x8e, 0x1d, 0x93, 0x50, 0x66, 0xd8, 0x0e, - 0xbb, 0xaa, 0x3a, 0x8c, 0xd0, 0x0b, 0xa3, 0x25, 0xb7, 0xeb, 0x39, 0xcc, 0x53, 0x62, 0x58, 0x7a, - 0x2f, 0x82, 0x54, 0x08, 0xac, 0x84, 0x26, 0xe6, 0x61, 0xb6, 0x7e, 0xde, 0x32, 0xd7, 0x1b, 0x81, - 0x92, 0xba, 0x28, 0x59, 0x6e, 0xda, 0x13, 0x22, 0x0c, 0x88, 0x74, 0x6d, 0x8f, 0xd9, 0x4e, 0x33, - 0x84, 0x37, 0x39, 0x3e, 0xde, 0x42, 0x60, 0xde, 0xc7, 0xdc, 0x82, 0x6c, 0xdb, 0xe8, 0x86, 0xe0, - 0x92, 0x63, 0xc0, 0xe1, 0xb9, 0xb6, 0xd1, 0xed, 0x63, 0x7c, 0x03, 0x77, 0xdc, 0x53, 0x8f, 0xd0, - 0x0b, 0x12, 0x5a, 0xa7, 0xa7, 0xa5, 0x56, 0x93, 0x31, 0x01, 0x5a, 0x57, 0xda, 0xb7, 0xfd, 0x43, - 0xee, 0xed, 0x01, 0xaf, 0x1f, 0x03, 0x0d, 0x6a, 0x38, 0x9e, 0x61, 0x32, 0xdb, 0x75, 0x4a, 0xa7, - 0x22, 0x14, 0xc2, 0x31, 0x60, 0xc0, 0x62, 0x48, 0xe1, 0xd0, 0xf7, 0xd4, 0x75, 0xdd, 0x06, 0xe8, - 0xf8, 0xde, 0x39, 0x21, 0x3a, 0xeb, 0x3a, 0x6a, 0xef, 0x0b, 0x11, 0x2e, 0x85, 0x8c, 0x83, 0x28, - 0x95, 0x76, 0x8d, 0xae, 0xa3, 0xa6, 0x38, 0x83, 0xbb, 0x21, 0x2d, 0x4c, 0x18, 0xbd, 0x92, 0x73, - 0xec, 0xc2, 0x34, 0x25, 0x86, 0xe7, 0x4a, 0xfc, 0xf9, 0xcd, 0x47, 0xa3, 0xf1, 0x85, 0x25, 0x16, - 0x06, 0x6a, 0x2a, 0x65, 0xae, 0xe6, 0x29, 0xc0, 0xd2, 0x80, 0x76, 0xa7, 0x65, 0x5c, 0x85, 0x97, - 0xfa, 0xa7, 0xc4, 0x80, 0xc2, 0x11, 0x33, 0x98, 0xef, 0x49, 0x4f, 0x96, 0x20, 0xc9, 0x93, 0x4d, - 0x22, 0x94, 0x6c, 0xb8, 0x00, 0xd5, 0x7a, 0x1e, 0x4e, 0x0a, 0x0f, 0x3f, 0x1d, 0xed, 0x61, 0x08, - 0x72, 0x3d, 0xca, 0xd1, 0xb5, 0xcf, 0x60, 0x5a, 0xca, 0x11, 0x82, 0x79, 0x5c, 0x29, 0x1d, 0xd5, - 0x6b, 0xfa, 0x71, 0x6d, 0xaf, 0x56, 0x7f, 0x51, 0xcb, 0x4f, 0x20, 0x0d, 0x16, 0x95, 0xac, 0xf1, - 0xb2, 0xa6, 0xd7, 0xea, 0x0d, 0x7d, 0xa7, 0x7e, 0x5c, 0x2b, 0xe7, 0x13, 0x6a, 0x01, 0x27, 0x90, - 0x7f, 0x41, 0x6d, 0x46, 0x78, 0xa4, 0x38, 0x32, 0x93, 0xa1, 0xcf, 0x21, 0x6d, 0x8b, 0xbf, 0x9e, - 0x96, 0x10, 0xf7, 0xe6, 0x5e, 0x84, 0x8b, 0xd2, 0x20, 0x28, 0x0c, 0x4a, 0x5f, 0x82, 0x3e, 0x4f, - 0xcd, 0x4c, 0xe6, 0x93, 0x6b, 0xbf, 0x4d, 0x28, 0xec, 0x86, 0xeb, 0xd6, 0x5b, 0x2a, 0x13, 0x96, - 0x60, 0xf6, 0xbd, 0xc2, 0xaf, 0x6f, 0x85, 0x6a, 0x90, 0x37, 0x4c, 0xe6, 0x1b, 0xad, 0xf7, 0x0b, - 0xbc, 0x9c, 0x34, 0xee, 0x89, 0xd5, 0x46, 0x2c, 0x03, 0xaa, 0x77, 0x78, 0x42, 0xb7, 0x29, 0xf1, - 0x1a, 0x5d, 0x27, 0x7c, 0xca, 0x27, 0xb0, 0xb8, 0xed, 0x3a, 0x96, 0xcd, 0xcf, 0x63, 0xc7, 0xb0, - 0x5b, 0xea, 0xba, 0xa3, 0x1f, 0xc0, 0x9c, 0xf2, 0xe4, 0xc2, 0x68, 0xf9, 0x44, 0xad, 0x27, 0xaa, - 0x1a, 0x7d, 0xc5, 0xc7, 0x71, 0x46, 0x6a, 0x8b, 0x3f, 0x0a, 0xfa, 0xf7, 0x09, 0x40, 0xb2, 0x54, - 0x91, 0x6f, 0x89, 0x19, 0x04, 0x12, 0x2a, 0x40, 0xba, 0x4d, 0x3c, 0xcf, 0x68, 0x92, 0x81, 0x0b, - 0x14, 0x08, 0xd1, 0x17, 0x30, 0xab, 0x92, 0x34, 0xb1, 0xd4, 0xe2, 0x63, 0x8b, 0x60, 0xb0, 0x83, - 0x3d, 0x03, 0xf4, 0x0c, 0x66, 0x82, 0xec, 0xa3, 0x72, 0xcc, 0xbb, 0x8c, 0x7b, 0xfa, 0xca, 0xed, - 0xef, 0xc3, 0xec, 0x11, 0x71, 0xc6, 0x73, 0x76, 0xe0, 0x52, 0x5c, 0xc2, 0x62, 0xa9, 0x7d, 0x6a, - 0x37, 0x7d, 0xd7, 0xf7, 0x30, 0xf1, 0xfc, 0x16, 0x1b, 0x6f, 0xc1, 0x9f, 0x43, 0xe6, 0x92, 0x1a, - 0x9d, 0x0e, 0xb1, 0x74, 0x42, 0xe9, 0x88, 0x25, 0x0b, 0x38, 0x0c, 0x4a, 0xb9, 0x42, 0x83, 0x33, - 0x5c, 0xe1, 0x45, 0xf4, 0x8c, 0xed, 0x52, 0xd7, 0xef, 0x94, 0x49, 0x8b, 0xdc, 0xca, 0x59, 0x04, - 0x96, 0x14, 0x55, 0xd9, 0x76, 0x29, 0xf5, 0x3b, 0xfc, 0xa8, 0xa5, 0x67, 0x1f, 0xc1, 0xac, 0x60, - 0x7b, 0xfa, 0xed, 0x68, 0x9e, 0x11, 0xe2, 0x03, 0xaf, 0x89, 0xd6, 0x60, 0xb6, 0x43, 0x5d, 0x93, - 0x78, 0x9e, 0x3a, 0x8d, 0x99, 0x5e, 0xde, 0x0a, 0xc4, 0xbd, 0x9b, 0x84, 0xd4, 0x34, 0xe1, 0xa0, - 0xf8, 0x21, 0x80, 0x62, 0x56, 0x01, 0x41, 0x98, 0xda, 0x2a, 0x28, 0x82, 0x30, 0xab, 0xf4, 0x05, - 0x45, 0xe8, 0xff, 0xe1, 0xc7, 0x29, 0x3f, 0xad, 0x3e, 0xf3, 0x38, 0x62, 0x2e, 0x1d, 0x66, 0x1e, - 0x9c, 0x5e, 0x46, 0x31, 0x0f, 0xa1, 0x2d, 0x99, 0x87, 0xfa, 0xc4, 0x69, 0xa1, 0xdb, 0x83, 0x3c, - 0xe0, 0x6c, 0xe7, 0xdc, 0x70, 0xac, 0x16, 0xe7, 0x3a, 0x8c, 0x5e, 0xf5, 0xe8, 0x10, 0xda, 0x84, - 0x54, 0xa7, 0x42, 0xe9, 0x88, 0x2b, 0x2f, 0xf4, 0xd4, 0x3e, 0x08, 0xdd, 0xb5, 0xbf, 0x25, 0x40, - 0xfb, 0xf2, 0x16, 0x5a, 0x10, 0x69, 0xb1, 0xe9, 0xf2, 0x1b, 0x98, 0x66, 0x5d, 0x87, 0xbb, 0x2f, - 0x59, 0x4a, 0x99, 0x0f, 0xfd, 0xf5, 0xba, 0xf8, 0xa4, 0x69, 0xb3, 0x73, 0xff, 0x74, 0xdd, 0x74, - 0xdb, 0x1b, 0xbd, 0xc9, 0xad, 0xd3, 0xfe, 0xf7, 0x46, 0xe7, 0x55, 0x73, 0x43, 0xb0, 0x66, 0xdf, - 0xb7, 0xad, 0xf5, 0xe3, 0xe3, 0x6a, 0xf9, 0xe6, 0xba, 0x38, 0xd5, 0xe8, 0x3a, 0xd5, 0x32, 0x9e, - 0x62, 0x5d, 0xa7, 0x6a, 0xa1, 0x1d, 0xc8, 0xb0, 0x7e, 0xaa, 0x55, 0xb1, 0x30, 0x5e, 0x49, 0x0a, - 0x1b, 0xaa, 0xed, 0xba, 0x0f, 0x77, 0x1a, 0x5d, 0xe7, 0x90, 0x92, 0x8b, 0x12, 0x63, 0xa4, 0xdd, - 0x19, 0x20, 0x86, 0x9f, 0x40, 0xb1, 0xd1, 0x75, 0x4a, 0x2d, 0x4e, 0x23, 0xae, 0x2a, 0x8e, 0xe9, - 0xfa, 0x9c, 0x9b, 0xa8, 0x4b, 0x18, 0x56, 0xfc, 0x45, 0x02, 0x16, 0x79, 0x72, 0x6d, 0x12, 0x5a, - 0xbf, 0x20, 0xf4, 0xac, 0xe5, 0x5e, 0xca, 0x1d, 0xba, 0x07, 0xc9, 0x08, 0x7e, 0xc7, 0x65, 0xe8, - 0x11, 0x64, 0x4d, 0x9f, 0x52, 0xe2, 0x30, 0x95, 0x89, 0x26, 0xc5, 0x51, 0x4b, 0x4f, 0xe7, 0xd4, - 0x90, 0x48, 0x3b, 0xe8, 0xbb, 0x90, 0xb3, 0x1d, 0x93, 0x92, 0x76, 0x5f, 0x39, 0x19, 0x52, 0x9e, - 0xef, 0x0d, 0x86, 0xb3, 0xd4, 0x01, 0x2c, 0x1c, 0xd8, 0x5d, 0x62, 0x1d, 0xf9, 0x26, 0xbf, 0xce, - 0xc1, 0x15, 0x48, 0xab, 0x28, 0x7b, 0xd7, 0x2d, 0xc0, 0x81, 0xa2, 0x82, 0xfb, 0x4d, 0x02, 0xee, - 0x6f, 0x71, 0x4e, 0xd8, 0xcf, 0xcd, 0xe4, 0xcc, 0xa5, 0x64, 0x77, 0xbb, 0x57, 0x24, 0x1a, 0xef, - 0x55, 0x24, 0xfa, 0x3c, 0x88, 0x43, 0x9c, 0x53, 0xe2, 0xf1, 0x06, 0xe5, 0x7f, 0xa9, 0x0e, 0x7d, - 0x2b, 0xe5, 0xeb, 0x4b, 0x40, 0xb2, 0xd4, 0x1d, 0xd8, 0x9e, 0x67, 0x3b, 0x4d, 0xe9, 0xe1, 0x17, - 0x30, 0x77, 0x49, 0x5d, 0xa7, 0xa9, 0xcb, 0xc2, 0xa7, 0x9c, 0x8c, 0xaf, 0x93, 0x38, 0x23, 0xd4, - 0xe5, 0x1f, 0x85, 0xfc, 0xaf, 0x3b, 0x90, 0x11, 0x68, 0x65, 0xc2, 0x0c, 0xbb, 0x85, 0x0e, 0x21, - 0xef, 0xb8, 0x4c, 0x1f, 0x68, 0xac, 0x24, 0xee, 0xff, 0x47, 0xe0, 0x46, 0x34, 0x77, 0x78, 0xde, - 0x19, 0x10, 0xa2, 0x03, 0xc8, 0xc9, 0xb6, 0x83, 0xe3, 0x9e, 0xf1, 0xa4, 0xa0, 0xb6, 0xe2, 0xff, - 0xe2, 0xa8, 0xf6, 0x40, 0xf2, 0xc0, 0x59, 0x1a, 0x96, 0xa1, 0xaf, 0x00, 0x49, 0xb8, 0x57, 0xe4, - 0x4a, 0x0f, 0x68, 0xbd, 0x0a, 0x9a, 0x87, 0x71, 0x88, 0xb7, 0x9b, 0x16, 0x9c, 0xa7, 0xb7, 0xc4, - 0xe8, 0xa7, 0xb0, 0x2a, 0x78, 0xf9, 0xa5, 0xa0, 0xef, 0xba, 0xdf, 0xe7, 0xef, 0x62, 0x7b, 0x39, - 0x81, 0x57, 0x2d, 0xc2, 0xd3, 0xc8, 0x0e, 0xf3, 0x5d, 0xc4, 0x1f, 0xaf, 0xd0, 0x51, 0x3a, 0x9c, - 0x33, 0x87, 0x82, 0x59, 0x37, 0x24, 0xa3, 0x15, 0x6d, 0x62, 0x66, 0xf3, 0xf1, 0xe8, 0x6c, 0x10, - 0xa6, 0xbf, 0x18, 0xb1, 0xa1, 0x01, 0x84, 0x21, 0x1f, 0x06, 0xe7, 0x44, 0x56, 0x9b, 0x16, 0xc8, - 0x9f, 0x8c, 0x46, 0xee, 0xf1, 0x66, 0x9c, 0x63, 0x83, 0x52, 0x74, 0x0c, 0x0b, 0x61, 0x4c, 0xca, - 0x93, 0xa9, 0x96, 0x8e, 0x3d, 0x87, 0x48, 0xa6, 0x8c, 0xc3, 0x6e, 0x09, 0x31, 0x7a, 0x09, 0xe1, - 0x05, 0xf0, 0x96, 0x92, 0xf9, 0x9e, 0x36, 0x23, 0x70, 0x1f, 0x8d, 0xcd, 0x52, 0x71, 0xd8, 0x37, - 0x29, 0x47, 0x3b, 0x3c, 0x5c, 0x6c, 0x46, 0x82, 0x70, 0x99, 0x15, 0x98, 0x0f, 0x22, 0x30, 0x6f, - 0x93, 0x51, 0x1e, 0x38, 0x3d, 0x09, 0xda, 0x85, 0xac, 0xc4, 0x61, 0xae, 0xab, 0xf3, 0xc8, 0x86, - 0xd1, 0x40, 0xa1, 0x22, 0xab, 0x80, 0xa4, 0x84, 0x47, 0x86, 0xdb, 0xd1, 0xa9, 0xa2, 0x7b, 0xa2, - 0x1f, 0xc9, 0xc4, 0x46, 0xc6, 0x30, 0x2f, 0xc4, 0x59, 0x37, 0x2c, 0xe3, 0x87, 0x6c, 0x06, 0x04, - 0x51, 0x3f, 0x13, 0x0c, 0x51, 0x9b, 0x8b, 0x3d, 0xe4, 0x28, 0x2e, 0x89, 0x73, 0xe6, 0xa0, 0x14, - 0xed, 0xc3, 0xbc, 0x4c, 0x05, 0x54, 0x31, 0x43, 0x2d, 0x1b, 0xeb, 0xe1, 0x30, 0x83, 0xc4, 0xd9, - 0x56, 0x58, 0xc6, 0x3d, 0x74, 0x5c, 0x8b, 0xe8, 0x7e, 0xff, 0x5d, 0x43, 0x9b, 0x8f, 0xf5, 0x30, - 0xea, 0x05, 0x04, 0xe7, 0x9c, 0x41, 0x29, 0xfa, 0x14, 0x52, 0x1e, 0x71, 0x2c, 0x2d, 0x27, 0x70, - 0x3e, 0x8c, 0xc0, 0xe9, 0x71, 0x44, 0x2c, 0x34, 0x65, 0x06, 0x39, 0x63, 0x7a, 0x93, 0xb3, 0x30, - 0xdd, 0x92, 0x34, 0x4c, 0xcb, 0x8f, 0xc8, 0x20, 0x11, 0x8c, 0x8d, 0x67, 0x90, 0x41, 0x31, 0xbf, - 0xb9, 0x01, 0x81, 0x32, 0x7b, 0xf4, 0x4d, 0x5b, 0x88, 0xbd, 0xb9, 0xd1, 0x54, 0x0f, 0x2f, 0xd0, - 0xdb, 0x72, 0x91, 0x42, 0x15, 0x72, 0x70, 0xe7, 0x50, 0x7c, 0x0a, 0x1d, 0xa2, 0x76, 0x38, 0x4b, - 0xc3, 0xb2, 0xdb, 0x21, 0x46, 0x45, 0x43, 0xa9, 0x2d, 0x8d, 0x13, 0x62, 0xa1, 0xe6, 0x73, 0x20, - 0xc4, 0xa4, 0x9c, 0x1f, 0xb0, 0x11, 0x10, 0x6b, 0x9d, 0x0a, 0x66, 0xad, 0x2d, 0xc7, 0x1e, 0x70, - 0x14, 0x07, 0xc7, 0x39, 0x63, 0x50, 0xca, 0x17, 0x2f, 0xc9, 0x63, 0xbf, 0x7e, 0xdc, 0x8f, 0x5d, - 0xfc, 0x30, 0xf9, 0xc4, 0x59, 0x2f, 0x2c, 0x43, 0xdf, 0xc2, 0x7d, 0x45, 0x26, 0x65, 0xca, 0xe2, - 0x97, 0x88, 0x87, 0x9e, 0x2e, 0x88, 0xb4, 0xf6, 0xa1, 0x80, 0xfe, 0x4e, 0x04, 0x74, 0x1c, 0x69, - 0xc4, 0xda, 0x79, 0x1c, 0x9d, 0x3c, 0x81, 0xbb, 0x1c, 0x59, 0xe5, 0x72, 0xdd, 0xf0, 0xae, 0x1c, - 0x53, 0x74, 0x0e, 0x85, 0xd8, 0x8a, 0x1a, 0xc1, 0xdd, 0x30, 0x62, 0xdd, 0x20, 0x93, 0x97, 0x38, - 0x44, 0x85, 0x52, 0xbe, 0xd3, 0xb6, 0xe4, 0x67, 0xba, 0xab, 0x08, 0x9a, 0x56, 0x8c, 0xdd, 0xe9, - 0x28, 0x2a, 0x87, 0x73, 0xf6, 0xa0, 0x94, 0x97, 0x20, 0xbf, 0xff, 0xae, 0xa8, 0xab, 0x56, 0x4d, - 0x5b, 0x8d, 0x2d, 0x41, 0x31, 0xaf, 0x90, 0x18, 0xf9, 0x43, 0x03, 0xa8, 0x0a, 0xd9, 0x36, 0x67, - 0x6f, 0xba, 0x27, 0xe9, 0x9b, 0xf6, 0x51, 0xec, 0x73, 0xed, 0x10, 0xcb, 0xc3, 0x73, 0xed, 0x90, - 0x08, 0x9d, 0x40, 0xbe, 0xd7, 0x74, 0xeb, 0xa7, 0x82, 0xb4, 0x69, 0x6b, 0x02, 0x6d, 0x3d, 0x02, - 0x6d, 0x04, 0xc7, 0xc3, 0x39, 0x36, 0x28, 0x47, 0x3e, 0xac, 0x88, 0x13, 0x93, 0x0c, 0x59, 0x27, - 0x7d, 0x8a, 0xac, 0xee, 0xc7, 0x03, 0x31, 0xcf, 0x66, 0xf4, 0xc9, 0x8d, 0x22, 0xd6, 0x78, 0x99, - 0xc5, 0x2a, 0xf0, 0x34, 0x2b, 0x8b, 0x12, 0x67, 0x34, 0x9c, 0xe0, 0x69, 0x1f, 0xc7, 0x5e, 0xf1, - 0x61, 0x22, 0x88, 0xb3, 0x76, 0x58, 0xf6, 0x6c, 0xe6, 0xcd, 0xeb, 0x62, 0x42, 0xb5, 0xbb, 0xf7, - 0xf2, 0xcb, 0xcf, 0x53, 0x33, 0x2b, 0xf9, 0xc2, 0xda, 0x86, 0x20, 0x7a, 0x87, 0xae, 0x27, 0xf2, - 0x3b, 0x5a, 0x86, 0x29, 0xdb, 0xb1, 0x48, 0x57, 0x75, 0x7a, 0x92, 0x78, 0x4a, 0x91, 0xa2, 0x86, - 0xbf, 0x4b, 0xc2, 0xd4, 0x78, 0x7d, 0xf1, 0x8f, 0x07, 0xb9, 0x0b, 0x25, 0xe2, 0x21, 0x58, 0x90, - 0xb2, 0xf9, 0xc8, 0x35, 0x0c, 0x64, 0x14, 0xa1, 0x1c, 0x3c, 0xf8, 0xb1, 0xa1, 0x11, 0xb4, 0x0d, - 0x59, 0xdf, 0x21, 0xdd, 0x8e, 0xeb, 0x11, 0x4b, 0x14, 0xc9, 0xd4, 0x38, 0x1d, 0x12, 0x9e, 0xeb, - 0x19, 0xf1, 0xe2, 0xb8, 0x01, 0x19, 0x97, 0xda, 0x4d, 0xdb, 0xd1, 0x79, 0x01, 0x11, 0xb4, 0x6a, - 0x6a, 0x6b, 0x9e, 0xcf, 0xf9, 0xf6, 0xba, 0x38, 0xcd, 0x4b, 0x4d, 0xb5, 0x8c, 0x41, 0xaa, 0xf0, - 0x7f, 0xe8, 0x33, 0x98, 0xb6, 0x04, 0x25, 0x56, 0x44, 0xa9, 0x10, 0xd7, 0x57, 0x48, 0xe2, 0x8c, - 0x95, 0x36, 0xfa, 0x5e, 0xb0, 0xaf, 0xe9, 0x51, 0x66, 0xc1, 0x31, 0xa8, 0x1d, 0x47, 0x4f, 0x21, - 0xe9, 0xb8, 0x97, 0x8a, 0xe6, 0x8c, 0xd5, 0x23, 0x70, 0xfd, 0x67, 0x33, 0xbf, 0x7c, 0x5d, 0x9c, - 0xe8, 0x3f, 0x6c, 0x3c, 0xfe, 0x63, 0xe2, 0xd6, 0x53, 0x61, 0xef, 0x61, 0x51, 0xbe, 0xc1, 0x35, - 0xf0, 0x89, 0x3e, 0xf4, 0x3a, 0xf7, 0x01, 0xdc, 0x91, 0x23, 0x2f, 0x70, 0xb5, 0x51, 0xd1, 0x1b, - 0xf5, 0xba, 0x5e, 0xdf, 0x2f, 0xe7, 0x13, 0x68, 0x09, 0x90, 0x1c, 0x28, 0x57, 0xf6, 0x2b, 0x8d, - 0x8a, 0x8e, 0x4b, 0xb5, 0xdd, 0x4a, 0x7e, 0xb2, 0x2f, 0x3f, 0xaa, 0xe0, 0x6a, 0x69, 0xbf, 0xfa, - 0x75, 0x69, 0x6b, 0xbf, 0x92, 0x4f, 0xa2, 0x7b, 0x70, 0x57, 0xca, 0x0f, 0xeb, 0x47, 0x47, 0xd5, - 0xad, 0xfd, 0x8a, 0x8e, 0x2b, 0x87, 0xfb, 0xa5, 0x93, 0x7c, 0x0a, 0xad, 0xc0, 0x3d, 0x39, 0x54, - 0x3a, 0x3a, 0xa9, 0x6d, 0xab, 0x99, 0x76, 0x4a, 0xd5, 0xfd, 0x63, 0x5c, 0xc9, 0x4f, 0x2d, 0xa7, - 0x7e, 0xf6, 0xeb, 0xc2, 0xc4, 0xe3, 0x67, 0x80, 0x86, 0x2f, 0x06, 0x9a, 0x81, 0x54, 0xad, 0x5e, - 0xab, 0xe4, 0x27, 0x50, 0x06, 0xd2, 0x5b, 0xa5, 0xed, 0xbd, 0xfa, 0xce, 0x4e, 0x3e, 0x81, 0xb2, - 0x30, 0x5b, 0x3d, 0x38, 0xa8, 0x94, 0xab, 0xa5, 0x46, 0x25, 0x3f, 0xb9, 0xf5, 0xd1, 0x9b, 0x7f, - 0x14, 0x26, 0xde, 0xdc, 0x14, 0x12, 0x7f, 0xbe, 0x29, 0x24, 0xfe, 0x72, 0x53, 0x48, 0xfc, 0xfd, - 0xa6, 0x90, 0xf8, 0xf9, 0x3f, 0x0b, 0x13, 0x5f, 0xa7, 0xd5, 0xb6, 0xff, 0x37, 0x00, 0x00, 0xff, - 0xff, 0xf8, 0xc7, 0xcb, 0x05, 0x38, 0x1b, 0x00, 0x00, + // 2495 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcd, 0x53, 0x23, 0xc7, + 0x15, 0x47, 0x48, 0x20, 0x78, 0x42, 0x48, 0xf4, 0xb2, 0x78, 0x96, 0x35, 0x12, 0x66, 0xed, 0x78, + 0xed, 0x54, 0xc0, 0x85, 0x63, 0xa7, 0xbc, 0x71, 0x0e, 0x02, 0x09, 0x2c, 0x83, 0x24, 0xd2, 0x08, + 0xef, 0x62, 0xa7, 0x6a, 0x6a, 0x98, 0x69, 0xc4, 0x78, 0xa5, 0x19, 0xa5, 0xa7, 0x07, 0xc4, 0x25, + 0xe7, 0x1c, 0x93, 0x5b, 0x6e, 0xd9, 0xaa, 0x9c, 0x52, 0xb9, 0xe4, 0x90, 0xca, 0x25, 0xa9, 0x9c, + 0xf7, 0x98, 0x63, 0x2a, 0x55, 0xa1, 0x12, 0x72, 0x49, 0x55, 0xfe, 0x83, 0x3d, 0xa5, 0xfa, 0x63, + 0xa4, 0x11, 0x9a, 0xd1, 0x2a, 0x7b, 0x92, 0xe6, 0xf5, 0x7b, 0xbf, 0x7e, 0xfd, 0xf1, 0xde, 0xfb, + 0xbd, 0x86, 0x65, 0xea, 0x1a, 0xe6, 0x45, 0xf7, 0x6c, 0x8b, 0x50, 0xea, 0x52, 0x6f, 0xb3, 0x4b, + 0x5d, 0xe6, 0xa2, 0x25, 0xd3, 0x35, 0x9f, 0x8b, 0x91, 0x4d, 0x35, 0xbe, 0x8a, 0x02, 0x45, 0xcb, + 0x60, 0x86, 0x54, 0x5b, 0x5d, 0x09, 0x64, 0x1d, 0xc2, 0x8c, 0x90, 0xfc, 0x91, 0xc7, 0x5c, 0x6a, + 0xb4, 0xc8, 0x16, 0x71, 0x5a, 0xb6, 0x13, 0xfc, 0x70, 0xbd, 0x4b, 0xd3, 0xfc, 0x58, 0x29, 0x69, + 0x3e, 0xb3, 0xdb, 0x5b, 0x17, 0x6d, 0x73, 0x8b, 0xd9, 0x1d, 0xe2, 0x31, 0xa3, 0xd3, 0x55, 0x23, + 0xcb, 0x2d, 0xb7, 0xe5, 0x8a, 0xbf, 0x5b, 0xfc, 0x9f, 0x94, 0x6e, 0xfc, 0x71, 0x1a, 0xee, 0xd5, + 0x5d, 0x76, 0x48, 0x0c, 0x8f, 0x7c, 0xe1, 0xb6, 0x2d, 0x42, 0x2b, 0xdc, 0x65, 0x54, 0x86, 0x34, + 0x25, 0xdd, 0xb6, 0x6d, 0x1a, 0x5a, 0x62, 0x3d, 0xf1, 0x38, 0xb3, 0xfd, 0xee, 0xe6, 0x88, 0xf7, + 0x9b, 0x58, 0x6a, 0x94, 0x89, 0x67, 0x52, 0xbb, 0xcb, 0x5c, 0xba, 0x93, 0x7a, 0x79, 0x53, 0x9c, + 0xc2, 0x81, 0x29, 0xda, 0x87, 0x85, 0x36, 0x47, 0xd6, 0x2f, 0x04, 0xb4, 0x36, 0x3d, 0x39, 0x14, + 0xce, 0xb4, 0x07, 0x3e, 0xa1, 0x4f, 0x60, 0x8e, 0x1a, 0x4e, 0x8b, 0xe8, 0xb6, 0xa5, 0x25, 0xd7, + 0x13, 0x8f, 0x93, 0x3b, 0xab, 0x7c, 0xa6, 0xdb, 0x9b, 0x62, 0x1a, 0x73, 0x79, 0xb5, 0xfc, 0x6a, + 0xf0, 0x17, 0xa7, 0x85, 0x6e, 0xd5, 0x42, 0x9b, 0x30, 0x23, 0x50, 0xb4, 0x94, 0x98, 0x58, 0x8b, + 0x98, 0x58, 0xac, 0x1c, 0x4b, 0x35, 0xf4, 0x08, 0xc0, 0xf4, 0x3d, 0xe6, 0x76, 0xf4, 0x8e, 0xd7, + 0xd2, 0x66, 0xd6, 0x13, 0x8f, 0xe7, 0xd5, 0x92, 0xe6, 0xa5, 0xbc, 0xe6, 0xb5, 0x9e, 0xa4, 0xfe, + 0xf3, 0xa2, 0x98, 0xd8, 0x78, 0x1b, 0x96, 0xeb, 0xae, 0x45, 0x4e, 0x1c, 0xe3, 0xd2, 0xb0, 0xdb, + 0xc6, 0x59, 0x9b, 0x88, 0x8d, 0x53, 0xa3, 0x45, 0x78, 0xeb, 0xc4, 0xf1, 0xfc, 0x6e, 0xd7, 0xa5, + 0x8c, 0x58, 0x98, 0xfc, 0xd4, 0x27, 0x1e, 0x0b, 0x2b, 0xfc, 0x18, 0x90, 0xf0, 0xb6, 0xee, 0xb2, + 0x3d, 0xd7, 0x77, 0x2c, 0xb9, 0xeb, 0xe1, 0x65, 0x26, 0x26, 0x5e, 0xa6, 0x82, 0xfc, 0xf5, 0x34, + 0xdc, 0x17, 0x43, 0x07, 0xe4, 0xba, 0x66, 0x7b, 0x1d, 0x83, 0x99, 0x17, 0x12, 0xf6, 0x63, 0x58, + 0xa2, 0xd2, 0x05, 0xdd, 0x63, 0x06, 0x65, 0xfa, 0x73, 0x72, 0x2d, 0xf0, 0x17, 0x76, 0xd2, 0xaf, + 0x6e, 0x8a, 0xc9, 0x03, 0x72, 0x8d, 0x73, 0x4a, 0xe3, 0x98, 0x2b, 0x1c, 0x90, 0x6b, 0xb4, 0x05, + 0x81, 0x48, 0x27, 0x8e, 0x25, 0x4c, 0xa6, 0x87, 0x4d, 0xb2, 0x6a, 0xbc, 0xe2, 0x58, 0xdc, 0xa0, + 0x06, 0xf9, 0x8e, 0x9a, 0x96, 0x58, 0xba, 0xf0, 0x4d, 0x9c, 0x55, 0x66, 0x7b, 0x23, 0xea, 0xc0, + 0xf9, 0x78, 0xe8, 0xb8, 0x73, 0x03, 0x5b, 0x31, 0x84, 0x0e, 0x20, 0xe7, 0xf9, 0xad, 0x16, 0xf1, + 0x58, 0x1f, 0x2d, 0x35, 0x31, 0xda, 0x62, 0xdf, 0x54, 0x8c, 0xa8, 0x1d, 0xfa, 0xef, 0x34, 0x6c, + 0x60, 0x62, 0x58, 0x4f, 0x6d, 0x76, 0x61, 0x3b, 0x27, 0x8e, 0x49, 0x28, 0x33, 0x6c, 0x87, 0x5d, + 0x57, 0x1d, 0x46, 0xe8, 0xa5, 0xd1, 0x96, 0xdb, 0xf5, 0x25, 0x2c, 0x52, 0x62, 0x58, 0x7a, 0x3f, + 0x82, 0x54, 0x08, 0xac, 0x85, 0x26, 0xe6, 0x61, 0xb6, 0x79, 0xd1, 0x36, 0x37, 0x9b, 0x81, 0x92, + 0xba, 0x28, 0x59, 0x6e, 0xda, 0x17, 0x22, 0x0c, 0x88, 0xf4, 0x6c, 0x8f, 0xd9, 0x4e, 0x2b, 0x84, + 0x37, 0x3d, 0x39, 0xde, 0x52, 0x60, 0x3e, 0xc0, 0xdc, 0x81, 0x6c, 0xc7, 0xe8, 0x85, 0xe0, 0x92, + 0x13, 0xc0, 0xe1, 0x85, 0x8e, 0xd1, 0x1b, 0x60, 0x7c, 0x03, 0xf7, 0xdc, 0x33, 0x8f, 0xd0, 0x4b, + 0x12, 0x5a, 0xa7, 0xa7, 0xa5, 0xd6, 0x93, 0x31, 0x01, 0xda, 0x50, 0xda, 0x77, 0xfd, 0x43, 0xee, + 0xdd, 0x01, 0x6f, 0x10, 0x03, 0x4d, 0x6a, 0x38, 0x9e, 0x61, 0x32, 0xdb, 0x75, 0x4a, 0x67, 0x22, + 0x14, 0xc2, 0x31, 0x60, 0xc0, 0x72, 0x48, 0xe1, 0xc8, 0xf7, 0xd4, 0x75, 0xdd, 0x05, 0xe8, 0xfa, + 0xde, 0x05, 0x21, 0x3a, 0xeb, 0x39, 0x6a, 0xef, 0x0b, 0x11, 0x2e, 0x85, 0x8c, 0x83, 0x28, 0x95, + 0x76, 0xcd, 0x9e, 0xa3, 0xa6, 0x38, 0x87, 0xfb, 0x21, 0x2d, 0x4c, 0x18, 0xbd, 0x96, 0x73, 0xec, + 0xc3, 0x2c, 0x25, 0x86, 0xe7, 0x4a, 0xfc, 0xc5, 0xed, 0x0f, 0xc6, 0xe3, 0x0b, 0x4b, 0x2c, 0x0c, + 0xd4, 0x54, 0xca, 0x5c, 0xcd, 0x53, 0x80, 0x95, 0x21, 0xed, 0x6e, 0xdb, 0xb8, 0x0e, 0x2f, 0xf5, + 0x2f, 0x89, 0x21, 0x85, 0x63, 0x66, 0x30, 0xdf, 0x93, 0x9e, 0xac, 0x40, 0x92, 0x27, 0x9b, 0x44, + 0x28, 0xd9, 0x70, 0x01, 0xaa, 0xf7, 0x3d, 0x9c, 0x16, 0x1e, 0x7e, 0x34, 0xde, 0xc3, 0x10, 0xe4, + 0x66, 0x94, 0xa3, 0x1b, 0x9f, 0xc2, 0xac, 0x94, 0x23, 0x04, 0x8b, 0xb8, 0x52, 0x3a, 0x6e, 0xd4, + 0xf5, 0x93, 0xfa, 0x41, 0xbd, 0xf1, 0xb4, 0x9e, 0x9f, 0x42, 0x1a, 0x2c, 0x2b, 0x59, 0xf3, 0x59, + 0x5d, 0xaf, 0x37, 0x9a, 0xfa, 0x5e, 0xe3, 0xa4, 0x5e, 0xce, 0x27, 0xd4, 0x02, 0x4e, 0x21, 0xff, + 0x94, 0xda, 0x8c, 0xf0, 0x48, 0x71, 0x64, 0x26, 0x43, 0x9f, 0x41, 0xda, 0x16, 0x9f, 0x9e, 0x96, + 0x10, 0xf7, 0xe6, 0x41, 0x84, 0x8b, 0xd2, 0x20, 0x28, 0x0c, 0x4a, 0x5f, 0x82, 0x7e, 0x99, 0x9a, + 0x9b, 0xce, 0x27, 0x37, 0x7e, 0x97, 0x50, 0xd8, 0x4d, 0xd7, 0x6d, 0xb4, 0x55, 0x26, 0x2c, 0xc1, + 0xfc, 0x1b, 0x85, 0xdf, 0xc0, 0x0a, 0xd5, 0x21, 0x6f, 0x98, 0xcc, 0x37, 0xda, 0x6f, 0x16, 0x78, + 0x39, 0x69, 0xdc, 0x17, 0xab, 0x8d, 0x58, 0x05, 0xd4, 0xe8, 0xf2, 0x84, 0x6e, 0x53, 0xe2, 0x35, + 0x7b, 0x4e, 0xf8, 0x94, 0x4f, 0x61, 0x79, 0xd7, 0x75, 0x2c, 0x9b, 0x9f, 0xc7, 0x9e, 0x61, 0xb7, + 0xd5, 0x75, 0x47, 0x3f, 0x84, 0x05, 0xe5, 0xc9, 0xa5, 0xd1, 0xf6, 0x89, 0x5a, 0x4f, 0x54, 0x35, + 0xfa, 0x8a, 0x8f, 0xe3, 0x8c, 0xd4, 0x16, 0x1f, 0x0a, 0xfa, 0x0f, 0x09, 0x40, 0xb2, 0x54, 0x91, + 0x6f, 0x89, 0x19, 0x04, 0x12, 0x2a, 0x40, 0xba, 0x43, 0x3c, 0xcf, 0x68, 0x91, 0xa1, 0x0b, 0x14, + 0x08, 0xd1, 0xe7, 0x30, 0xaf, 0x92, 0x34, 0xb1, 0xd4, 0xe2, 0x63, 0x8b, 0x60, 0xb0, 0x83, 0x7d, + 0x03, 0xf4, 0x04, 0xe6, 0x82, 0xec, 0xa3, 0x72, 0xcc, 0xeb, 0x8c, 0xfb, 0xfa, 0xca, 0xed, 0x1f, + 0xc0, 0xfc, 0x31, 0x71, 0x26, 0x73, 0x76, 0xe8, 0x52, 0x5c, 0xc1, 0x72, 0xa9, 0x73, 0x66, 0xb7, + 0x7c, 0xd7, 0xf7, 0x30, 0xf1, 0xfc, 0x36, 0x9b, 0x6c, 0xc1, 0x9f, 0x41, 0xe6, 0x8a, 0x1a, 0xdd, + 0x2e, 0xb1, 0x74, 0x42, 0xe9, 0x98, 0x25, 0x0b, 0x38, 0x0c, 0x4a, 0xb9, 0x42, 0x83, 0x33, 0x5c, + 0xe3, 0x45, 0xf4, 0x9c, 0xed, 0x53, 0xd7, 0xef, 0x96, 0x49, 0x9b, 0xdc, 0xc9, 0x59, 0x04, 0x56, + 0x14, 0x55, 0xd9, 0x75, 0x29, 0xf5, 0xbb, 0xfc, 0xa8, 0xa5, 0x67, 0xef, 0xc0, 0xbc, 0x60, 0x7b, + 0xfa, 0xdd, 0x68, 0x9e, 0x13, 0xe2, 0x9a, 0xd7, 0x42, 0x1b, 0x30, 0xdf, 0xa5, 0xae, 0x49, 0x3c, + 0x4f, 0x9d, 0xc6, 0x5c, 0x3f, 0x6f, 0x05, 0xe2, 0xfe, 0x4d, 0x42, 0x6a, 0x9a, 0x70, 0x50, 0xfc, + 0x08, 0x40, 0x31, 0xab, 0x80, 0x20, 0xcc, 0xec, 0x14, 0x14, 0x41, 0x98, 0x57, 0xfa, 0x82, 0x22, + 0x0c, 0x3e, 0xf8, 0x71, 0xca, 0xbf, 0xd6, 0x80, 0x79, 0x1c, 0x33, 0x97, 0x8e, 0x32, 0x0f, 0x4e, + 0x2f, 0xa3, 0x98, 0x87, 0xd0, 0x96, 0xcc, 0x43, 0xfd, 0xc5, 0x69, 0xa1, 0xdb, 0x87, 0xac, 0x71, + 0xb6, 0x73, 0x61, 0x38, 0x56, 0x9b, 0x73, 0x1d, 0x46, 0xaf, 0xfb, 0x74, 0x08, 0x6d, 0x43, 0xaa, + 0x5b, 0xa1, 0x74, 0xcc, 0x95, 0x17, 0x7a, 0x6a, 0x1f, 0x84, 0xee, 0xc6, 0x3f, 0x12, 0xa0, 0x7d, + 0x71, 0x07, 0x2d, 0x88, 0xb4, 0xd8, 0x74, 0xf9, 0x0d, 0xcc, 0xb2, 0x9e, 0xc3, 0xdd, 0x97, 0x2c, + 0xa5, 0xcc, 0x87, 0xfe, 0x7e, 0x53, 0xfc, 0xb8, 0x65, 0xb3, 0x0b, 0xff, 0x6c, 0xd3, 0x74, 0x3b, + 0x5b, 0xfd, 0xc9, 0xad, 0xb3, 0xc1, 0xff, 0xad, 0xee, 0xf3, 0xd6, 0x96, 0x60, 0xcd, 0xbe, 0x6f, + 0x5b, 0x9b, 0x27, 0x27, 0xd5, 0xf2, 0xed, 0x4d, 0x71, 0xa6, 0xd9, 0x73, 0xaa, 0x65, 0x3c, 0xc3, + 0x7a, 0x4e, 0xd5, 0x42, 0x7b, 0x90, 0x61, 0x83, 0x54, 0xab, 0x62, 0x61, 0xb2, 0x92, 0x14, 0x36, + 0x54, 0xdb, 0xf5, 0x10, 0xee, 0x35, 0x7b, 0xce, 0x11, 0x25, 0x97, 0x25, 0xc6, 0x48, 0xa7, 0x3b, + 0x44, 0x0c, 0xdf, 0x87, 0x62, 0xb3, 0xe7, 0x94, 0xda, 0x9c, 0x46, 0x5c, 0x57, 0x1c, 0xd3, 0xf5, + 0x39, 0x37, 0x51, 0x97, 0x30, 0xac, 0xf8, 0xcb, 0x04, 0x2c, 0xf3, 0xe4, 0xda, 0x22, 0xb4, 0x71, + 0x49, 0xe8, 0x79, 0xdb, 0xbd, 0x92, 0x3b, 0xf4, 0x00, 0x92, 0x11, 0xfc, 0x8e, 0xcb, 0xd0, 0x07, + 0x90, 0x35, 0x7d, 0x4a, 0x89, 0xc3, 0x54, 0x26, 0x9a, 0x16, 0x47, 0x2d, 0x3d, 0x5d, 0x50, 0x43, + 0x22, 0xed, 0xa0, 0xef, 0x41, 0xce, 0x76, 0x4c, 0x4a, 0x3a, 0x03, 0xe5, 0x64, 0x48, 0x79, 0xb1, + 0x3f, 0x18, 0xce, 0x52, 0x35, 0x58, 0xaa, 0xd9, 0x3d, 0x62, 0x1d, 0xfb, 0x26, 0xbf, 0xce, 0xc1, + 0x15, 0x48, 0xab, 0x28, 0x7b, 0xdd, 0x2d, 0xc0, 0x81, 0xa2, 0x82, 0xfb, 0x6d, 0x02, 0x1e, 0xee, + 0x70, 0x4e, 0x38, 0xc8, 0xcd, 0xe4, 0xdc, 0xa5, 0x64, 0x7f, 0xb7, 0x5f, 0x24, 0x9a, 0x6f, 0x54, + 0x24, 0x06, 0x3c, 0x88, 0x43, 0x5c, 0x50, 0xe2, 0xf1, 0x06, 0xe5, 0xff, 0xa9, 0x0e, 0x03, 0x2b, + 0xe5, 0xeb, 0x33, 0x40, 0xb2, 0xd4, 0xd5, 0x6c, 0xcf, 0xb3, 0x9d, 0x96, 0xf4, 0xf0, 0x73, 0x58, + 0xb8, 0xa2, 0xae, 0xd3, 0xd2, 0x65, 0xe1, 0x53, 0x4e, 0xc6, 0xd7, 0x49, 0x9c, 0x11, 0xea, 0xf2, + 0x63, 0xd0, 0x69, 0xd4, 0x08, 0x6d, 0x91, 0xaa, 0x73, 0x44, 0xdd, 0x16, 0x0d, 0xf6, 0x55, 0x8d, + 0xfe, 0x79, 0x19, 0x32, 0xe2, 0xbb, 0x4c, 0x98, 0x61, 0xb7, 0xd1, 0x11, 0xe4, 0x1d, 0x97, 0xe9, + 0x43, 0x6d, 0x97, 0x9c, 0xf5, 0x3b, 0x11, 0xb3, 0x46, 0xb4, 0x7e, 0x78, 0xd1, 0x19, 0x12, 0xa2, + 0x1a, 0xe4, 0x64, 0x53, 0xc2, 0x71, 0xcf, 0x79, 0xca, 0x50, 0x1b, 0xf5, 0x5e, 0x1c, 0x11, 0x1f, + 0x4a, 0x2d, 0x38, 0x4b, 0xc3, 0x32, 0xf4, 0x15, 0x20, 0x09, 0xf7, 0x9c, 0x5c, 0xeb, 0x01, 0xe9, + 0x57, 0x21, 0xf5, 0x38, 0x0e, 0xf1, 0x6e, 0x4b, 0x83, 0xf3, 0xf4, 0x8e, 0x18, 0xfd, 0x0c, 0xd6, + 0x05, 0x6b, 0xbf, 0x12, 0xe4, 0x5e, 0xf7, 0x07, 0xec, 0x5e, 0x6c, 0x3e, 0xa7, 0xf7, 0xaa, 0x81, + 0xf8, 0x24, 0xb2, 0xff, 0x7c, 0x5d, 0x5b, 0x80, 0xd7, 0xe8, 0x38, 0x1d, 0xce, 0xa8, 0x43, 0xa1, + 0xae, 0x1b, 0x92, 0xef, 0x8a, 0x26, 0x32, 0xb3, 0xfd, 0xe1, 0xf8, 0x5c, 0x11, 0x26, 0xc7, 0x18, + 0xb1, 0x91, 0x01, 0x84, 0x21, 0x1f, 0x06, 0xe7, 0x34, 0x57, 0x9b, 0x15, 0xc8, 0xef, 0x8f, 0x47, + 0xee, 0xb3, 0x6a, 0x9c, 0x63, 0xc3, 0x52, 0x74, 0x02, 0x4b, 0x61, 0x4c, 0xca, 0x53, 0xad, 0x96, + 0x8e, 0x3d, 0x87, 0x48, 0x1e, 0x8d, 0xc3, 0x6e, 0x09, 0x31, 0x7a, 0x06, 0xe1, 0x05, 0xf0, 0x86, + 0x93, 0xf9, 0x9e, 0x36, 0x27, 0x70, 0x3f, 0x98, 0x98, 0xc3, 0xe2, 0xb0, 0x6f, 0x52, 0x8e, 0xf6, + 0x78, 0x30, 0xd9, 0x8c, 0x04, 0xc1, 0x34, 0x2f, 0x30, 0x1f, 0x45, 0x60, 0xde, 0xa5, 0xaa, 0x3c, + 0xac, 0xfa, 0x12, 0xb4, 0x0f, 0x59, 0x89, 0xc3, 0x5c, 0x57, 0xe7, 0x71, 0x0f, 0xe3, 0x81, 0x42, + 0x25, 0x58, 0x01, 0x49, 0x09, 0x8f, 0x0c, 0xb7, 0xab, 0x53, 0x45, 0x06, 0x45, 0xb7, 0x92, 0x89, + 0x8d, 0x8c, 0x51, 0xd6, 0x88, 0xb3, 0x6e, 0x58, 0xc6, 0x0f, 0xd9, 0x0c, 0xe8, 0xa3, 0x7e, 0x2e, + 0xf8, 0xa3, 0xb6, 0x10, 0x7b, 0xc8, 0x51, 0x4c, 0x13, 0xe7, 0xcc, 0x61, 0x29, 0x3a, 0x84, 0x45, + 0x99, 0x0a, 0xa8, 0xe2, 0x8d, 0x5a, 0x36, 0xd6, 0xc3, 0x51, 0x7e, 0x89, 0xb3, 0xed, 0xb0, 0x8c, + 0x7b, 0xe8, 0xb8, 0x16, 0xd1, 0xfd, 0xc1, 0xab, 0x87, 0xb6, 0x18, 0xeb, 0x61, 0xd4, 0xfb, 0x08, + 0xce, 0x39, 0xc3, 0x52, 0xf4, 0x11, 0xa4, 0x3c, 0xe2, 0x58, 0x5a, 0x4e, 0xe0, 0xbc, 0x1d, 0x81, + 0xd3, 0x67, 0x90, 0x58, 0x68, 0xca, 0x0c, 0x72, 0xce, 0xf4, 0x16, 0xe7, 0x68, 0xba, 0x25, 0x49, + 0x9a, 0x96, 0x1f, 0x93, 0x41, 0x22, 0xf8, 0x1c, 0xcf, 0x20, 0xc3, 0x62, 0x7e, 0x73, 0x03, 0x7a, + 0x65, 0xf6, 0xc9, 0x9d, 0xb6, 0x14, 0x7b, 0x73, 0xa3, 0x89, 0x20, 0x5e, 0xa2, 0x77, 0xe5, 0x22, + 0x85, 0x2a, 0xe4, 0xe0, 0xce, 0xa1, 0xf8, 0x14, 0x3a, 0x42, 0xfc, 0x70, 0x96, 0x86, 0x65, 0x77, + 0x43, 0x8c, 0x8a, 0x76, 0x53, 0x5b, 0x99, 0x24, 0xc4, 0x42, 0xad, 0xe9, 0x50, 0x88, 0x49, 0x39, + 0x3f, 0x60, 0x23, 0xa0, 0xdd, 0x3a, 0x15, 0xbc, 0x5b, 0x5b, 0x8d, 0x3d, 0xe0, 0x28, 0x86, 0x8e, + 0x73, 0xc6, 0xb0, 0x94, 0x2f, 0x5e, 0x52, 0xcb, 0x41, 0xfd, 0x78, 0x18, 0xbb, 0xf8, 0x51, 0x6a, + 0x8a, 0xb3, 0x5e, 0x58, 0x86, 0xbe, 0x85, 0x87, 0x8a, 0x6a, 0xca, 0x94, 0xc5, 0x2f, 0x11, 0x0f, + 0x3d, 0x5d, 0xd0, 0x6c, 0xed, 0x6d, 0x01, 0xfd, 0xdd, 0x08, 0xe8, 0x38, 0x4a, 0x89, 0xb5, 0x8b, + 0x38, 0xb2, 0x79, 0x0a, 0xf7, 0x39, 0xb2, 0xca, 0xe5, 0xba, 0xe1, 0x5d, 0x3b, 0xa6, 0xe8, 0x2b, + 0x0a, 0xb1, 0x15, 0x35, 0x82, 0xd9, 0x61, 0xc4, 0x7a, 0x41, 0x26, 0x2f, 0x71, 0x88, 0x0a, 0xa5, + 0x7c, 0xa7, 0x6d, 0xc9, 0xde, 0x74, 0x57, 0xd1, 0x37, 0xad, 0x18, 0xbb, 0xd3, 0x51, 0x44, 0x0f, + 0xe7, 0xec, 0x61, 0x29, 0x2f, 0x41, 0xfe, 0xe0, 0xd5, 0x51, 0x57, 0x8d, 0x9c, 0xb6, 0x1e, 0x5b, + 0x82, 0x62, 0xde, 0x28, 0x31, 0xf2, 0x47, 0x06, 0x50, 0x15, 0xb2, 0x1d, 0xce, 0xed, 0x74, 0x4f, + 0x92, 0x3b, 0xed, 0x9d, 0xd8, 0xc7, 0xdc, 0x11, 0x0e, 0x88, 0x17, 0x3a, 0x21, 0x11, 0x3a, 0x85, + 0x7c, 0xbf, 0x25, 0xd7, 0xcf, 0x04, 0xa5, 0xd3, 0x36, 0x04, 0xda, 0x66, 0x04, 0xda, 0x18, 0x06, + 0x88, 0x73, 0x6c, 0x58, 0x8e, 0x7c, 0x58, 0x13, 0x27, 0x26, 0xf9, 0xb3, 0x4e, 0x06, 0x04, 0x5a, + 0xdd, 0x8f, 0x47, 0x62, 0x9e, 0xed, 0xe8, 0x93, 0x1b, 0x47, 0xbb, 0xf1, 0x2a, 0x8b, 0x55, 0xe0, + 0x69, 0x56, 0x16, 0x25, 0xce, 0x68, 0x38, 0xfd, 0xd3, 0xde, 0x8d, 0xbd, 0xe2, 0xa3, 0x34, 0x11, + 0x67, 0xed, 0xb0, 0x0c, 0x1d, 0xc3, 0x52, 0x87, 0x33, 0x3e, 0xdd, 0x76, 0xf4, 0xae, 0xe2, 0x7c, + 0xda, 0x7b, 0xb1, 0x97, 0x23, 0x8a, 0x1d, 0xe2, 0x5c, 0x67, 0x58, 0xfa, 0x64, 0xee, 0xe5, 0x8b, + 0x62, 0x42, 0x75, 0xd8, 0x0f, 0xf2, 0xab, 0x5f, 0xa6, 0xe6, 0xd6, 0xf2, 0x85, 0x8d, 0x2d, 0xc1, + 0x1e, 0x8f, 0x5c, 0x4f, 0x14, 0x0d, 0xb4, 0x0a, 0x33, 0xb6, 0x63, 0x91, 0x9e, 0x6a, 0x2e, 0x25, + 0xd7, 0x95, 0x22, 0xc5, 0x37, 0x7f, 0x9f, 0x84, 0x99, 0xc9, 0x5a, 0xf1, 0x9f, 0x0c, 0x13, 0x22, + 0x4a, 0xc4, 0xdb, 0xb3, 0x60, 0x7a, 0x8b, 0x91, 0x1b, 0x33, 0x94, 0xa6, 0x84, 0x72, 0xf0, 0xc6, + 0xc8, 0x46, 0x46, 0xd0, 0x2e, 0x64, 0x7d, 0x87, 0xf4, 0xba, 0xae, 0x47, 0x2c, 0x51, 0x79, 0x53, + 0x93, 0x34, 0x65, 0x78, 0xa1, 0x6f, 0xc4, 0x2b, 0xee, 0x16, 0x64, 0x5c, 0x6a, 0xb7, 0x6c, 0x47, + 0xe7, 0x55, 0x49, 0x70, 0xb5, 0x99, 0x9d, 0x45, 0x3e, 0xe7, 0xab, 0x9b, 0xe2, 0x2c, 0xaf, 0x5f, + 0xd5, 0x32, 0x06, 0xa9, 0xc2, 0xbf, 0xd0, 0xa7, 0x30, 0x6b, 0x09, 0x9e, 0xad, 0xd8, 0x57, 0x21, + 0xae, 0x95, 0x91, 0x6c, 0x1c, 0x2b, 0x6d, 0xf4, 0xfd, 0x60, 0x5f, 0xd3, 0xe3, 0xcc, 0x82, 0x63, + 0x50, 0x3b, 0x8e, 0x3e, 0x81, 0xa4, 0xe3, 0x5e, 0x29, 0xee, 0x34, 0x51, 0x5b, 0xc2, 0xf5, 0x9f, + 0xcc, 0xfd, 0xea, 0x45, 0x71, 0x6a, 0xf0, 0x96, 0xf2, 0xe1, 0x9f, 0x12, 0x77, 0x5e, 0x27, 0xfb, + 0x6f, 0x99, 0xf2, 0xd9, 0xaf, 0x89, 0x4f, 0xf5, 0x91, 0x07, 0xc1, 0xb7, 0xe0, 0x9e, 0x1c, 0x79, + 0x8a, 0xab, 0xcd, 0x8a, 0xde, 0x6c, 0x34, 0xf4, 0xc6, 0x61, 0x39, 0x9f, 0x40, 0x2b, 0x80, 0xe4, + 0x40, 0xb9, 0x72, 0x58, 0x69, 0x56, 0x74, 0x5c, 0xaa, 0xef, 0x57, 0xf2, 0xd3, 0x03, 0xf9, 0x71, + 0x05, 0x57, 0x4b, 0x87, 0xd5, 0xaf, 0x4b, 0x3b, 0x87, 0x95, 0x7c, 0x12, 0x3d, 0x80, 0xfb, 0x52, + 0x7e, 0xd4, 0x38, 0x3e, 0xae, 0xee, 0x1c, 0x56, 0x74, 0x5c, 0x39, 0x3a, 0x2c, 0x9d, 0xe6, 0x53, + 0x68, 0x0d, 0x1e, 0xc8, 0xa1, 0xd2, 0xf1, 0x69, 0x7d, 0x57, 0xcd, 0xb4, 0x57, 0xaa, 0x1e, 0x9e, + 0xe0, 0x4a, 0x7e, 0x66, 0x35, 0xf5, 0xf3, 0xdf, 0x14, 0xa6, 0x3e, 0x7c, 0x02, 0x68, 0xf4, 0x62, + 0xa0, 0x39, 0x48, 0xd5, 0x1b, 0xf5, 0x4a, 0x7e, 0x0a, 0x65, 0x20, 0xbd, 0x53, 0xda, 0x3d, 0x68, + 0xec, 0xed, 0xe5, 0x13, 0x28, 0x0b, 0xf3, 0xd5, 0x5a, 0xad, 0x52, 0xae, 0x96, 0x9a, 0x95, 0xfc, + 0xf4, 0xce, 0x3b, 0x2f, 0xff, 0x55, 0x98, 0x7a, 0x79, 0x5b, 0x48, 0xfc, 0xf5, 0xb6, 0x90, 0xf8, + 0xdb, 0x6d, 0x21, 0xf1, 0xcf, 0xdb, 0x42, 0xe2, 0x17, 0xff, 0x2e, 0x4c, 0x7d, 0x9d, 0x56, 0xdb, + 0xfe, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x53, 0xfd, 0xe2, 0xb1, 0xab, 0x1b, 0x00, 0x00, } diff --git a/pkg/roachpb/errors.proto b/pkg/roachpb/errors.proto index 06c8175b45f0..48dcabbc6479 100644 --- a/pkg/roachpb/errors.proto +++ b/pkg/roachpb/errors.proto @@ -394,6 +394,15 @@ message IntentMissingError { optional Intent wrong_intent = 1; } +// A MergeInProgressError indicates that the request could not be completed +// because the replica is being merged into its left-hand neighbor. The request +// should be resubmitted after the merge completes. +// +// This error is handled by the Store and should not escape to higher levels. +message MergeInProgressError { + option (gogoproto.equal) = true; +} + // ErrorDetail is a union type containing all available errors. message ErrorDetail { option (gogoproto.equal) = true; @@ -435,6 +444,7 @@ message ErrorDetail { optional BatchTimestampBeforeGCError timestamp_before = 34; optional TxnAlreadyEncounteredErrorError txn_already_encountered_error = 35; optional IntentMissingError intent_missing = 36; + optional MergeInProgressError merge_in_progress = 37; } // TransactionRestart indicates how an error should be handled in a diff --git a/pkg/storage/client_merge_test.go b/pkg/storage/client_merge_test.go index bbc2ea2330a2..1ce131e981ca 100644 --- a/pkg/storage/client_merge_test.go +++ b/pkg/storage/client_merge_test.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "math/rand" "reflect" "strings" "sync/atomic" @@ -35,6 +36,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/storage/storagebase" "github.com/cockroachdb/cockroach/pkg/storage/txnwait" "github.com/cockroachdb/cockroach/pkg/testutils" + "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -194,8 +196,6 @@ func TestStoreRangeMergeMetadataCleanup(t *testing.T) { func TestStoreRangeMergeWithData(t *testing.T) { defer leaktest.AfterTest(t)() - t.Skip("#27442") - for _, colocate := range []bool{false, true} { for _, retries := range []int64{0, 3} { t.Run(fmt.Sprintf("colocate=%v/retries=%d", colocate, retries), func(t *testing.T) { @@ -883,6 +883,116 @@ func TestStoreRangeMergeRHSLeaseExpiration(t *testing.T) { } } +// TestStoreRangeMergeConcurrentRequests tests merging ranges that are serving +// other traffic concurrently. +func TestStoreRangeMergeConcurrentRequests(t *testing.T) { + defer leaktest.AfterTest(t)() + + ctx := context.Background() + storeCfg := storage.TestStoreConfig(nil) + storeCfg.TestingKnobs.DisableSplitQueue = true + storeCfg.TestingKnobs.DisableReplicateQueue = true + + var mtc *multiTestContext + storeCfg.TestingKnobs.TestingRequestFilter = func(ba roachpb.BatchRequest) *roachpb.Error { + if ba.IsSingleGetSnapshotForMergeRequest() && rand.Int()%4 == 0 { + // Before every few GetSnapshotForMerge requests, expire all range leases. + // This makes the following sequence of events quite likely: + // + // 1. The merge transaction begins and lays down deletion intents for + // the meta2 and local copies of the RHS range descriptor. + // 2. The RHS replica loses its lease, thanks to the following call to + // mtc.advanceClock. + // 3. A Get request arrives at the RHS replica and triggers a + // synchronous lease acquisition. The lease acquisition notices + // that a merge is in progress and installs a mergeComplete + // channel. + // 4. The Get request blocks on the newly installed mergeComplete + // channel. + // 5. The GetSnapshotForMerge request arrives. + // + // This scenario previously caused deadlock. The merge was not able to + // complete until the GetSnapshotForMerge request completed, but the + // GetSnapshotForMerge request was stuck in the command queue until the + // Get request finished, which was itself waiting for the merge to + // complete. Whoops! + mtc.advanceClock(ctx) + } + return nil + } + + mtc = &multiTestContext{storeConfig: &storeCfg} + mtc.Start(t, 1) + defer mtc.Stop() + store := mtc.Store(0) + + keys := []roachpb.Key{ + roachpb.Key("a1"), roachpb.Key("a2"), roachpb.Key("a3"), + roachpb.Key("c1"), roachpb.Key("c2"), roachpb.Key("c3"), + } + + for _, k := range keys { + if err := store.DB().Put(ctx, k, "val"); err != nil { + t.Fatal(err) + } + } + + // Failures in this test often present as a deadlock. Set a short timeout to + // limit the damage. + ctx, cancel := context.WithTimeout(ctx, testutils.DefaultSucceedsSoonDuration) + defer cancel() + + const numGetWorkers = 16 + const numMerges = 16 + + var numGets int64 + doneCh := make(chan struct{}) + g := ctxgroup.WithContext(ctx) + for i := 0; i < numGetWorkers; i++ { + g.GoCtx(func(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-doneCh: + return nil + default: + } + key := keys[rand.Intn(len(keys))] + if kv, err := store.DB().Get(ctx, key); err != nil { + return err + } else if v := string(kv.ValueBytes()); v != "val" { + return fmt.Errorf(`expected "val", but got %q`, v) + } + atomic.AddInt64(&numGets, 1) + } + }) + } + + for i := 0; i < numMerges; i++ { + lhsDesc, _, err := createSplitRanges(ctx, store) + if err != nil { + t.Fatal(err) + } + args := adminMergeArgs(lhsDesc.StartKey.AsRawKey()) + if _, pErr := client.SendWrapped(ctx, store.TestSender(), args); pErr != nil { + t.Fatal(pErr) + } + } + + close(doneCh) + if err := g.Wait(); err != nil { + t.Fatal(err) + } + + // Expect that each worker was able to issue one at least one get request + // during every split/merge cycle. Empirical evidence suggests that this a + // very conservative estimate that is unlikely to be flaky. + if n := atomic.LoadInt64(&numGets); n < numGetWorkers*numMerges { + t.Fatalf("suspiciously low numGets (expected at least %d): %d", numGetWorkers*numMerges, n) + } +} + func TestInvalidGetSnapshotForMergeRequest(t *testing.T) { defer leaktest.AfterTest(t)() diff --git a/pkg/storage/replica.go b/pkg/storage/replica.go index e677ec842131..e52e0e166a97 100644 --- a/pkg/storage/replica.go +++ b/pkg/storage/replica.go @@ -1726,6 +1726,13 @@ func (r *Replica) getReplicaDescriptorRLocked() (roachpb.ReplicaDescriptor, erro return roachpb.ReplicaDescriptor{}, roachpb.NewRangeNotFoundError(r.RangeID) } +func (r *Replica) getMergeCompleteCh() chan struct{} { + r.mu.RLock() + mergeCompleteCh := r.mu.mergeComplete + r.mu.RUnlock() + return mergeCompleteCh +} + // setLastReplicaDescriptors sets the the most recently seen replica // descriptors to those contained in the *RaftMessageRequest, acquiring r.mu // to do so. @@ -2361,10 +2368,7 @@ func (r *Replica) beginCmds( fn(ba, storagebase.CommandQueueBeginExecuting) } - r.mu.RLock() - mergeCompleteCh := r.mu.mergeComplete - r.mu.RUnlock() - if mergeCompleteCh != nil && !ba.IsSingleGetSnapshotForMergeRequest() { + if r.getMergeCompleteCh() != nil && !ba.IsSingleGetSnapshotForMergeRequest() { // The replica is being merged into its left-hand neighbor. This request // cannot proceed until the merge completes, signaled by the closing of // the channel. @@ -2402,14 +2406,17 @@ func (r *Replica) beginCmds( // merge transaction, and merge transactions read the RHS descriptor at // the beginning of the transaction to verify that it has not already been // merged away. - select { - case <-mergeCompleteCh: - // Merge complete. Carry on. - case <-ctx.Done(): - return nil, ctx.Err() - case <-r.store.stopper.ShouldQuiesce(): - return nil, &roachpb.NodeUnavailableError{} - } + // + // We can't wait for the merge to complete here, though. The replica might + // need to respond to a GetSnapshotForMerge request in order for the merge + // to complete, and blocking here would force that GetSnapshotForMerge + // request to sit in the command queue forever, deadlocking the merge. + // Instead, we remove the commands from the command queue and return a + // MergeInProgressError. The store will catch that error and resubmit the + // request after mergeCompleteCh closes. See #27442 for the full context. + log.Event(ctx, "waiting on in-progress merge") + r.removeCmdsFromCommandQueue(newCmds) + return nil, &roachpb.MergeInProgressError{} } } else { log.Event(ctx, "operation accepts inconsistent results") diff --git a/pkg/storage/store.go b/pkg/storage/store.go index 1cc9e00c2d79..ad4550df8a9e 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -3038,6 +3038,24 @@ func (s *Store) Send( } // We've resolved the write intent; retry command. } + + case *roachpb.MergeInProgressError: + // A merge was in progress. We need to retry the command after the merge + // completes, as signaled by the closing of the replica's mergeComplete + // channel. Note that the merge may have already completed, in which case + // its mergeComplete channel will be nil. + mergeCompleteCh := repl.getMergeCompleteCh() + if mergeCompleteCh != nil { + select { + case <-mergeCompleteCh: + // Merge complete. Retry the command. + case <-ctx.Done(): + return nil, roachpb.NewError(ctx.Err()) + case <-s.stopper.ShouldQuiesce(): + return nil, roachpb.NewError(&roachpb.NodeUnavailableError{}) + } + } + pErr = nil } if pErr != nil {