From 3fdfdbb479d2f886006f01e18aa8d413052e5217 Mon Sep 17 00:00:00 2001 From: mangoslicer Date: Mon, 7 Aug 2017 01:59:18 -0400 Subject: [PATCH 1/2] Added initial code to fragment large watch messages --- clientv3/integration/fragment_test.go | 137 +++++++ clientv3/watch.go | 20 +- etcdserver/api/v3rpc/watch.go | 61 ++- etcdserver/etcdserverpb/rpc.pb.go | 522 +++++++++++++++----------- etcdserver/etcdserverpb/rpc.proto | 4 + proxy/grpcproxy/watch.go | 34 +- 6 files changed, 545 insertions(+), 233 deletions(-) create mode 100644 clientv3/integration/fragment_test.go diff --git a/clientv3/integration/fragment_test.go b/clientv3/integration/fragment_test.go new file mode 100644 index 00000000000..d2e99ee905a --- /dev/null +++ b/clientv3/integration/fragment_test.go @@ -0,0 +1,137 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integration + +import ( + "context" + "fmt" + "log" + "testing" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/integration" + "github.com/coreos/etcd/pkg/testutil" + "github.com/etcd/proxy/grpcproxy" + "github.com/etcd/proxy/grpcproxy/adapter" +) + +// TestFragmentationStopsAfterServerFailure tests the edge case where either +// the server of watch proxy fails to send a message due to errors not related to +// the fragment size, such as a member failure. In that case, +// the server or watch proxy should continue to reduce the message size (a default +// action choosen because there is no way of telling whether the error caused +// by the send operation is message size related or caused by some other issue) +// until the message size becomes zero and thus we are certain that the message +// size is not the issue causing the send operation to fail. +func TestFragmentationStopsAfterServerFailure(t *testing.T) { + defer testutil.AfterTest(t) + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + cfg := clientv3.Config{ + Endpoints: []string{ + clus.Members[0].GRPCAddr(), + clus.Members[1].GRPCAddr(), + clus.Members[2].GRPCAddr(), + }, + } + cli, err := clientv3.New(cfg) + + cli.SetEndpoints(clus.Members[0].GRPCAddr()) + firstLease, err := clus.Client(0).Grant(context.Background(), 10000) + if err != nil { + t.Error(err) + } + + kv := clus.Client(0) + for i := 0; i < 25; i++ { + _, err = kv.Put(context.TODO(), fmt.Sprintf("foo%d", i), "bar", clientv3.WithLease(firstLease.ID)) + if err != nil { + t.Error(err) + } + } + + kv.Watch(context.TODO(), "foo", clientv3.WithRange("z")) + _, err = clus.Client(0).Revoke(context.Background(), firstLease.ID) + if err != nil { + t.Error(err) + } + clus.Members[0].Stop(t) + time.Sleep(10 * time.Second) + log.Fatal("Printed the log") +} + +// TestFragmentingWithOverlappingWatchers tests that events are fragmented +// on the server and watch proxy and pieced back together on the client-side +// properly. +func TestFragmentingWithOverlappingWatchers(t *testing.T) { + defer testutil.AfterTest(t) + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + cfg := clientv3.Config{ + Endpoints: []string{ + clus.Members[0].GRPCAddr(), + clus.Members[1].GRPCAddr(), + clus.Members[2].GRPCAddr(), + }, + } + cli, err := clientv3.New(cfg) + + cli.SetEndpoints(clus.Members[0].GRPCAddr()) + firstLease, err := clus.Client(0).Grant(context.Background(), 10000) + if err != nil { + t.Error(err) + } + secondLease, err := clus.Client(0).Grant(context.Background(), 10000) + if err != nil { + t.Error(err) + } + + // Create and register watch proxy + wp, _ := grpcproxy.NewWatchProxy(clus.Client(0)) + wc := adapter.WatchServerToWatchClient(wp) + w := clientv3.NewWatchFromWatchClient(wc) + + kv := clus.Client(0) + for i := 0; i < 25; i++ { + _, err = kv.Put(context.TODO(), fmt.Sprintf("foo%d", i), "bar", clientv3.WithLease(firstLease.ID)) + if err != nil { + t.Error(err) + } + _, err = kv.Put(context.TODO(), fmt.Sprintf("buzz%d", i), "fizz", clientv3.WithLease(secondLease.ID)) + if err != nil { + t.Error(err) + } + } + + w.Watch(context.TODO(), "foo", clientv3.WithRange("z")) + w.Watch(context.TODO(), "buzz", clientv3.WithRange("z ")) + + _, err = clus.Client(0).Revoke(context.Background(), firstLease.ID) + if err != nil { + t.Error(err) + } + _, err = clus.Client(0).Revoke(context.Background(), secondLease.ID) + if err != nil { + t.Error(err) + } + + // Wait for the revokation process to finish + time.Sleep(10 * time.Second) + log.Fatal("Printed the log") + +} diff --git a/clientv3/watch.go b/clientv3/watch.go index ee43b2afeba..0f8ea25e1a6 100644 --- a/clientv3/watch.go +++ b/clientv3/watch.go @@ -424,7 +424,7 @@ func (w *watchGrpcStream) run() { } cancelSet := make(map[int64]struct{}) - + var combinedFragments *pb.WatchResponse for { select { // Watch() requested @@ -450,11 +450,24 @@ func (w *watchGrpcStream) run() { } // New events from the watch client case pbresp := <-w.respc: + fmt.Printf("Clientv3/watcher - watchid: %d, count: %d, frag: %d, events: %v\n", pbresp.WatchId, pbresp.FragmentCount, pbresp.CurrFragment, pbresp.Events) + if combinedFragments == nil || (combinedFragments.WatchId != pbresp.WatchId && combinedFragments.CurrFragment+1 != pbresp.CurrFragment) { + combinedFragments = pbresp + } else { + combinedFragments.Events = append(combinedFragments.Events, pbresp.Events...) + combinedFragments.CurrFragment = pbresp.CurrFragment + } + if combinedFragments.FragmentCount != combinedFragments.CurrFragment { + fmt.Printf("Waiting for more fragments ...\n") + break + } + fmt.Println("All fragments receieved\n") switch { case pbresp.Created: // response to head of queue creation if ws := w.resuming[0]; ws != nil { - w.addSubstream(pbresp, ws) + w.addSubstream(combinedFragments, ws) + combinedFragments = nil w.dispatchEvent(pbresp) w.resuming[0] = nil } @@ -470,7 +483,8 @@ func (w *watchGrpcStream) run() { } default: // dispatch to appropriate watch stream - if ok := w.dispatchEvent(pbresp); ok { + if ok := w.dispatchEvent(combinedFragments); ok { + combinedFragments = nil break } // watch response on unexpected watch id; cancel id diff --git a/etcdserver/api/v3rpc/watch.go b/etcdserver/api/v3rpc/watch.go index 84c0a5eac8c..1c07f517d70 100644 --- a/etcdserver/api/v3rpc/watch.go +++ b/etcdserver/api/v3rpc/watch.go @@ -15,7 +15,9 @@ package v3rpc import ( + "fmt" "io" + "math" "sync" "time" @@ -320,7 +322,6 @@ func (sws *serverWatchStream) sendLoop() { } } } - wr := &pb.WatchResponse{ Header: sws.newResponseHeader(wresp.Revision), WatchId: int64(wresp.WatchID), @@ -336,8 +337,35 @@ func (sws *serverWatchStream) sendLoop() { } mvcc.ReportEventReceived(len(evs)) - if err := sws.gRPCStream.Send(wr); err != nil { - return + // The grpc package's defaultServerMaxSendMessageSize, defaultServerMaxReceiveMessageSize + // and the maxSendMesgSize are set to lower numbers in order to demonstrate + // the fragmenting. + maxEventsPerMsg := 20 + watchRespSent := false + for !watchRespSent { + // There isn't a reasonable way of deciphering the cause of the send error. + // One solution that comes to mind is comparing the string error message + // to the expected string for a grpc message indicating that the + // message size is too large, but this approach would fail if grpc + // were to change their error message semantics in the future. The + // approach here is to assume that the error is caused by the message + // being too large and incrementally increasing the number of fragments + // until the maxEventsPerMesg becomes 0 and we are therefore sure + // that the error in the send operation was not caused by the message + // size. + if maxEventsPerMsg == 0 { + return + } + for _, fragment := range FragmentWatchResponse(maxEventsPerMsg, wr) { + if err := sws.gRPCStream.Send(fragment); err != nil { + fmt.Printf("Was about to size out. Setting max to %d from %d\n %v\n", maxEventsPerMsg/2, maxEventsPerMsg, err) + maxEventsPerMsg /= 2 + watchRespSent = false + break + } + fmt.Printf("etcdserver - watchid: %d, count: %d, frag: %d, events: %v\n", fragment.WatchId, fragment.FragmentCount, fragment.CurrFragment, fragment.Events) + watchRespSent = true + } } sws.mu.Lock() @@ -388,6 +416,33 @@ func (sws *serverWatchStream) sendLoop() { } } +func FragmentWatchResponse(maxSendMesgSize int, wr *pb.WatchResponse) []*pb.WatchResponse { + var fragmentWrs []*pb.WatchResponse + totalFragments := int64(math.Ceil(float64(len(wr.Events)) / float64(maxSendMesgSize))) + currFragmentCount := 1 + for i := 0; i < len(wr.Events); i += maxSendMesgSize { + eventRangeEnd := i + maxSendMesgSize + if eventRangeEnd > len(wr.Events) { + eventRangeEnd = len(wr.Events) + } + wresp := &pb.WatchResponse{ + Header: wr.Header, + WatchId: int64(wr.WatchId), + Events: wr.Events[i:eventRangeEnd], + CompactRevision: wr.CompactRevision, + FragmentCount: int64(totalFragments), + CurrFragment: int64(currFragmentCount), + } + currFragmentCount++ + fmt.Printf("Apending: total: %d, curr: %d\n", wresp.FragmentCount, wresp.CurrFragment) + fragmentWrs = append(fragmentWrs, wresp) + } + if len(fragmentWrs) == 0 { + return []*pb.WatchResponse{wr} + } + return fragmentWrs +} + func (sws *serverWatchStream) close() { sws.watchStream.Close() close(sws.closec) diff --git a/etcdserver/etcdserverpb/rpc.pb.go b/etcdserver/etcdserverpb/rpc.pb.go index 1cef5dc9e20..44528f71b00 100644 --- a/etcdserver/etcdserverpb/rpc.pb.go +++ b/etcdserver/etcdserverpb/rpc.pb.go @@ -1624,8 +1624,10 @@ type WatchResponse struct { // watcher with the same start_revision again. CompactRevision int64 `protobuf:"varint,5,opt,name=compact_revision,json=compactRevision,proto3" json:"compact_revision,omitempty"` // cancel_reason indicates the reason for canceling the watcher. - CancelReason string `protobuf:"bytes,6,opt,name=cancel_reason,json=cancelReason,proto3" json:"cancel_reason,omitempty"` - Events []*mvccpb.Event `protobuf:"bytes,11,rep,name=events" json:"events,omitempty"` + CancelReason string `protobuf:"bytes,6,opt,name=cancel_reason,json=cancelReason,proto3" json:"cancel_reason,omitempty"` + Events []*mvccpb.Event `protobuf:"bytes,11,rep,name=events" json:"events,omitempty"` + FragmentCount int64 `protobuf:"varint,7,opt,name=fragment_count,json=fragmentCount,proto3" json:"fragment_count,omitempty"` + CurrFragment int64 `protobuf:"varint,8,opt,name=curr_fragment,json=currFragment,proto3" json:"curr_fragment,omitempty"` } func (m *WatchResponse) Reset() { *m = WatchResponse{} } @@ -1682,6 +1684,20 @@ func (m *WatchResponse) GetEvents() []*mvccpb.Event { return nil } +func (m *WatchResponse) GetFragmentCount() int64 { + if m != nil { + return m.FragmentCount + } + return 0 +} + +func (m *WatchResponse) GetCurrFragment() int64 { + if m != nil { + return m.CurrFragment + } + return 0 +} + type LeaseGrantRequest struct { // TTL is the advisory time-to-live in seconds. TTL int64 `protobuf:"varint,1,opt,name=TTL,proto3" json:"TTL,omitempty"` @@ -5729,6 +5745,16 @@ func (m *WatchResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRpc(dAtA, i, uint64(len(m.CancelReason))) i += copy(dAtA[i:], m.CancelReason) } + if m.FragmentCount != 0 { + dAtA[i] = 0x38 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.FragmentCount)) + } + if m.CurrFragment != 0 { + dAtA[i] = 0x40 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.CurrFragment)) + } if len(m.Events) > 0 { for _, msg := range m.Events { dAtA[i] = 0x5a @@ -8072,6 +8098,12 @@ func (m *WatchResponse) Size() (n int) { if l > 0 { n += 1 + l + sovRpc(uint64(l)) } + if m.FragmentCount != 0 { + n += 1 + sovRpc(uint64(m.FragmentCount)) + } + if m.CurrFragment != 0 { + n += 1 + sovRpc(uint64(m.CurrFragment)) + } if len(m.Events) > 0 { for _, e := range m.Events { l = e.Size() @@ -11956,6 +11988,44 @@ func (m *WatchResponse) Unmarshal(dAtA []byte) error { } m.CancelReason = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FragmentCount", wireType) + } + m.FragmentCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FragmentCount |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CurrFragment", wireType) + } + m.CurrFragment = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CurrFragment |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } case 11: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Events", wireType) @@ -17586,227 +17656,229 @@ var ( func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 3549 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0x5f, 0x6f, 0x1b, 0xc7, - 0xb5, 0xd7, 0x92, 0x22, 0x29, 0x1e, 0xfe, 0x11, 0x35, 0x92, 0x6d, 0x6a, 0x6d, 0xcb, 0xf2, 0xf8, - 0x9f, 0x6c, 0xc7, 0x52, 0xa2, 0xe4, 0xde, 0x07, 0xdf, 0x20, 0xb8, 0xb2, 0xc4, 0x58, 0x8a, 0x64, - 0xc9, 0x59, 0xc9, 0x4e, 0x2e, 0x10, 0x5c, 0x62, 0x45, 0x8e, 0xa5, 0x85, 0xc8, 0x5d, 0x66, 0x77, - 0x49, 0x4b, 0x69, 0x0a, 0x14, 0x69, 0x82, 0xa2, 0x05, 0xfa, 0xd2, 0x3c, 0xf4, 0xdf, 0x63, 0x51, - 0x14, 0xf9, 0x00, 0x45, 0x3f, 0x40, 0x81, 0xa2, 0xe8, 0x4b, 0x0b, 0xf4, 0x0b, 0x14, 0x69, 0xbf, - 0x46, 0xd1, 0x62, 0xfe, 0xed, 0xce, 0x2e, 0x77, 0x25, 0x25, 0x6c, 0xf2, 0x62, 0xed, 0x9c, 0x39, - 0x73, 0x7e, 0x67, 0xce, 0xcc, 0x39, 0x67, 0xe6, 0x0c, 0x0d, 0x45, 0xb7, 0xd7, 0x5a, 0xec, 0xb9, - 0x8e, 0xef, 0xa0, 0x32, 0xf1, 0x5b, 0x6d, 0x8f, 0xb8, 0x03, 0xe2, 0xf6, 0xf6, 0xf5, 0x99, 0x03, - 0xe7, 0xc0, 0x61, 0x1d, 0x4b, 0xf4, 0x8b, 0xf3, 0xe8, 0xb3, 0x94, 0x67, 0xa9, 0x3b, 0x68, 0xb5, - 0xd8, 0x3f, 0xbd, 0xfd, 0xa5, 0xa3, 0x81, 0xe8, 0xba, 0xcc, 0xba, 0xcc, 0xbe, 0x7f, 0xc8, 0xfe, - 0xe9, 0xed, 0xb3, 0x3f, 0xa2, 0xf3, 0xca, 0x81, 0xe3, 0x1c, 0x74, 0xc8, 0x92, 0xd9, 0xb3, 0x96, - 0x4c, 0xdb, 0x76, 0x7c, 0xd3, 0xb7, 0x1c, 0xdb, 0xe3, 0xbd, 0xf8, 0x33, 0x0d, 0xaa, 0x06, 0xf1, - 0x7a, 0x8e, 0xed, 0x91, 0x75, 0x62, 0xb6, 0x89, 0x8b, 0xae, 0x02, 0xb4, 0x3a, 0x7d, 0xcf, 0x27, - 0x6e, 0xd3, 0x6a, 0xd7, 0xb5, 0x79, 0x6d, 0x61, 0xdc, 0x28, 0x0a, 0xca, 0x46, 0x1b, 0x5d, 0x86, - 0x62, 0x97, 0x74, 0xf7, 0x79, 0x6f, 0x86, 0xf5, 0x4e, 0x70, 0xc2, 0x46, 0x1b, 0xe9, 0x30, 0xe1, - 0x92, 0x81, 0xe5, 0x59, 0x8e, 0x5d, 0xcf, 0xce, 0x6b, 0x0b, 0x59, 0x23, 0x68, 0xd3, 0x81, 0xae, - 0xf9, 0xc2, 0x6f, 0xfa, 0xc4, 0xed, 0xd6, 0xc7, 0xf9, 0x40, 0x4a, 0xd8, 0x23, 0x6e, 0x17, 0x7f, - 0x9a, 0x83, 0xb2, 0x61, 0xda, 0x07, 0xc4, 0x20, 0x1f, 0xf6, 0x89, 0xe7, 0xa3, 0x1a, 0x64, 0x8f, - 0xc8, 0x09, 0x83, 0x2f, 0x1b, 0xf4, 0x93, 0x8f, 0xb7, 0x0f, 0x48, 0x93, 0xd8, 0x1c, 0xb8, 0x4c, - 0xc7, 0xdb, 0x07, 0xa4, 0x61, 0xb7, 0xd1, 0x0c, 0xe4, 0x3a, 0x56, 0xd7, 0xf2, 0x05, 0x2a, 0x6f, - 0x44, 0xd4, 0x19, 0x8f, 0xa9, 0xb3, 0x0a, 0xe0, 0x39, 0xae, 0xdf, 0x74, 0xdc, 0x36, 0x71, 0xeb, - 0xb9, 0x79, 0x6d, 0xa1, 0xba, 0x7c, 0x73, 0x51, 0x5d, 0x88, 0x45, 0x55, 0xa1, 0xc5, 0x5d, 0xc7, - 0xf5, 0x77, 0x28, 0xaf, 0x51, 0xf4, 0xe4, 0x27, 0x7a, 0x1b, 0x4a, 0x4c, 0x88, 0x6f, 0xba, 0x07, - 0xc4, 0xaf, 0xe7, 0x99, 0x94, 0x5b, 0x67, 0x48, 0xd9, 0x63, 0xcc, 0x06, 0x83, 0xe7, 0xdf, 0x08, - 0x43, 0xd9, 0x23, 0xae, 0x65, 0x76, 0xac, 0x8f, 0xcc, 0xfd, 0x0e, 0xa9, 0x17, 0xe6, 0xb5, 0x85, - 0x09, 0x23, 0x42, 0xa3, 0xf3, 0x3f, 0x22, 0x27, 0x5e, 0xd3, 0xb1, 0x3b, 0x27, 0xf5, 0x09, 0xc6, - 0x30, 0x41, 0x09, 0x3b, 0x76, 0xe7, 0x84, 0x2d, 0x9a, 0xd3, 0xb7, 0x7d, 0xde, 0x5b, 0x64, 0xbd, - 0x45, 0x46, 0x61, 0xdd, 0x0b, 0x50, 0xeb, 0x5a, 0x76, 0xb3, 0xeb, 0xb4, 0x9b, 0x81, 0x41, 0x80, - 0x19, 0xa4, 0xda, 0xb5, 0xec, 0x27, 0x4e, 0xdb, 0x90, 0x66, 0xa1, 0x9c, 0xe6, 0x71, 0x94, 0xb3, - 0x24, 0x38, 0xcd, 0x63, 0x95, 0x73, 0x11, 0xa6, 0xa9, 0xcc, 0x96, 0x4b, 0x4c, 0x9f, 0x84, 0xcc, - 0x65, 0xc6, 0x3c, 0xd5, 0xb5, 0xec, 0x55, 0xd6, 0x13, 0xe1, 0x37, 0x8f, 0x87, 0xf8, 0x2b, 0x82, - 0xdf, 0x3c, 0x8e, 0xf2, 0xe3, 0x45, 0x28, 0x06, 0x36, 0x47, 0x13, 0x30, 0xbe, 0xbd, 0xb3, 0xdd, - 0xa8, 0x8d, 0x21, 0x80, 0xfc, 0xca, 0xee, 0x6a, 0x63, 0x7b, 0xad, 0xa6, 0xa1, 0x12, 0x14, 0xd6, - 0x1a, 0xbc, 0x91, 0xc1, 0x8f, 0x00, 0x42, 0xeb, 0xa2, 0x02, 0x64, 0x37, 0x1b, 0xff, 0x57, 0x1b, - 0xa3, 0x3c, 0xcf, 0x1b, 0xc6, 0xee, 0xc6, 0xce, 0x76, 0x4d, 0xa3, 0x83, 0x57, 0x8d, 0xc6, 0xca, - 0x5e, 0xa3, 0x96, 0xa1, 0x1c, 0x4f, 0x76, 0xd6, 0x6a, 0x59, 0x54, 0x84, 0xdc, 0xf3, 0x95, 0xad, - 0x67, 0x8d, 0xda, 0x38, 0xfe, 0x5c, 0x83, 0x8a, 0x58, 0x2f, 0xee, 0x13, 0xe8, 0x0d, 0xc8, 0x1f, - 0x32, 0xbf, 0x60, 0x5b, 0xb1, 0xb4, 0x7c, 0x25, 0xb6, 0xb8, 0x11, 0xdf, 0x31, 0x04, 0x2f, 0xc2, - 0x90, 0x3d, 0x1a, 0x78, 0xf5, 0xcc, 0x7c, 0x76, 0xa1, 0xb4, 0x5c, 0x5b, 0xe4, 0x0e, 0xbb, 0xb8, - 0x49, 0x4e, 0x9e, 0x9b, 0x9d, 0x3e, 0x31, 0x68, 0x27, 0x42, 0x30, 0xde, 0x75, 0x5c, 0xc2, 0x76, - 0xec, 0x84, 0xc1, 0xbe, 0xe9, 0x36, 0x66, 0x8b, 0x26, 0x76, 0x2b, 0x6f, 0xe0, 0x2f, 0x34, 0x80, - 0xa7, 0x7d, 0x3f, 0xdd, 0x35, 0x66, 0x20, 0x37, 0xa0, 0x82, 0x85, 0x5b, 0xf0, 0x06, 0xf3, 0x09, - 0x62, 0x7a, 0x24, 0xf0, 0x09, 0xda, 0x40, 0x97, 0xa0, 0xd0, 0x73, 0xc9, 0xa0, 0x79, 0x34, 0x60, - 0x20, 0x13, 0x46, 0x9e, 0x36, 0x37, 0x07, 0xe8, 0x3a, 0x94, 0xad, 0x03, 0xdb, 0x71, 0x49, 0x93, - 0xcb, 0xca, 0xb1, 0xde, 0x12, 0xa7, 0x31, 0xbd, 0x15, 0x16, 0x2e, 0x38, 0xaf, 0xb2, 0x6c, 0x51, - 0x12, 0xb6, 0xa1, 0xc4, 0x54, 0x1d, 0xc9, 0x7c, 0x77, 0x43, 0x1d, 0x33, 0x6c, 0xd8, 0xb0, 0x09, - 0x85, 0xd6, 0xf8, 0x03, 0x40, 0x6b, 0xa4, 0x43, 0x7c, 0x32, 0x4a, 0xf4, 0x50, 0x6c, 0x92, 0x55, - 0x6d, 0x82, 0x7f, 0xa2, 0xc1, 0x74, 0x44, 0xfc, 0x48, 0xd3, 0xaa, 0x43, 0xa1, 0xcd, 0x84, 0x71, - 0x0d, 0xb2, 0x86, 0x6c, 0xa2, 0xfb, 0x30, 0x21, 0x14, 0xf0, 0xea, 0xd9, 0x94, 0x4d, 0x53, 0xe0, - 0x3a, 0x79, 0xf8, 0x8b, 0x0c, 0x14, 0xc5, 0x44, 0x77, 0x7a, 0x68, 0x05, 0x2a, 0x2e, 0x6f, 0x34, - 0xd9, 0x7c, 0x84, 0x46, 0x7a, 0x7a, 0x10, 0x5a, 0x1f, 0x33, 0xca, 0x62, 0x08, 0x23, 0xa3, 0xff, - 0x81, 0x92, 0x14, 0xd1, 0xeb, 0xfb, 0xc2, 0xe4, 0xf5, 0xa8, 0x80, 0x70, 0xff, 0xad, 0x8f, 0x19, - 0x20, 0xd8, 0x9f, 0xf6, 0x7d, 0xb4, 0x07, 0x33, 0x72, 0x30, 0x9f, 0x8d, 0x50, 0x23, 0xcb, 0xa4, - 0xcc, 0x47, 0xa5, 0x0c, 0x2f, 0xd5, 0xfa, 0x98, 0x81, 0xc4, 0x78, 0xa5, 0x53, 0x55, 0xc9, 0x3f, - 0xe6, 0xc1, 0x7b, 0x48, 0xa5, 0xbd, 0x63, 0x7b, 0x58, 0xa5, 0xbd, 0x63, 0xfb, 0x51, 0x11, 0x0a, - 0xa2, 0x85, 0x7f, 0x97, 0x01, 0x90, 0xab, 0xb1, 0xd3, 0x43, 0x6b, 0x50, 0x75, 0x45, 0x2b, 0x62, - 0xad, 0xcb, 0x89, 0xd6, 0x12, 0x8b, 0x38, 0x66, 0x54, 0xe4, 0x20, 0xae, 0xdc, 0x5b, 0x50, 0x0e, - 0xa4, 0x84, 0x06, 0x9b, 0x4d, 0x30, 0x58, 0x20, 0xa1, 0x24, 0x07, 0x50, 0x93, 0xbd, 0x07, 0x17, - 0x82, 0xf1, 0x09, 0x36, 0xbb, 0x7e, 0x8a, 0xcd, 0x02, 0x81, 0xd3, 0x52, 0x82, 0x6a, 0x35, 0x55, - 0xb1, 0xd0, 0x6c, 0xb3, 0x09, 0x66, 0x1b, 0x56, 0x8c, 0x1a, 0x0e, 0x68, 0xbe, 0xe4, 0x4d, 0xfc, - 0x87, 0x2c, 0x14, 0x56, 0x9d, 0x6e, 0xcf, 0x74, 0xe9, 0x6a, 0xe4, 0x5d, 0xe2, 0xf5, 0x3b, 0x3e, - 0x33, 0x57, 0x75, 0xf9, 0x46, 0x54, 0xa2, 0x60, 0x93, 0x7f, 0x0d, 0xc6, 0x6a, 0x88, 0x21, 0x74, - 0xb0, 0x48, 0x8f, 0x99, 0x73, 0x0c, 0x16, 0xc9, 0x51, 0x0c, 0x91, 0x8e, 0x9c, 0x0d, 0x1d, 0x59, - 0x87, 0xc2, 0x80, 0xb8, 0x61, 0x4a, 0x5f, 0x1f, 0x33, 0x24, 0x01, 0xdd, 0x85, 0xc9, 0x78, 0x7a, - 0xc9, 0x09, 0x9e, 0x6a, 0x2b, 0x9a, 0x8d, 0x6e, 0x40, 0x39, 0x92, 0xe3, 0xf2, 0x82, 0xaf, 0xd4, - 0x55, 0x52, 0xdc, 0x45, 0x19, 0x57, 0x69, 0x3e, 0x2e, 0xaf, 0x8f, 0xc9, 0xc8, 0x1a, 0x09, 0x26, - 0x13, 0xd1, 0x60, 0x82, 0xff, 0x17, 0x2a, 0x11, 0x43, 0xd0, 0xfc, 0xd2, 0x78, 0xf7, 0xd9, 0xca, - 0x16, 0x4f, 0x46, 0x8f, 0x59, 0xfe, 0x31, 0x6a, 0x1a, 0xcd, 0x69, 0x5b, 0x8d, 0xdd, 0xdd, 0x5a, - 0x06, 0x55, 0xa0, 0xb8, 0xbd, 0xb3, 0xd7, 0xe4, 0x5c, 0x59, 0xfc, 0x66, 0x20, 0x41, 0x24, 0x33, - 0x25, 0x87, 0x8d, 0x29, 0x39, 0x4c, 0x93, 0x39, 0x2c, 0x13, 0xe6, 0xb0, 0xec, 0xa3, 0x2a, 0x94, - 0xb9, 0xf1, 0x9a, 0x7d, 0x9b, 0xe6, 0xd1, 0x5f, 0x69, 0x00, 0xa1, 0xab, 0xa0, 0x25, 0x28, 0xb4, - 0xb8, 0xf0, 0xba, 0xc6, 0x22, 0xcd, 0x85, 0xc4, 0xf5, 0x30, 0x24, 0x17, 0x7a, 0x0d, 0x0a, 0x5e, - 0xbf, 0xd5, 0x22, 0x9e, 0xcc, 0x67, 0x97, 0xe2, 0xc1, 0x4e, 0x84, 0x22, 0x43, 0xf2, 0xd1, 0x21, - 0x2f, 0x4c, 0xab, 0xd3, 0x67, 0xd9, 0xed, 0xf4, 0x21, 0x82, 0x0f, 0xff, 0x5c, 0x83, 0x92, 0xb2, - 0x33, 0xbf, 0x66, 0x84, 0xbd, 0x02, 0x45, 0xa6, 0x03, 0x69, 0x8b, 0x18, 0x3b, 0x61, 0x84, 0x04, - 0xf4, 0xdf, 0x50, 0x94, 0xdb, 0x5b, 0x86, 0xd9, 0x7a, 0xb2, 0xd8, 0x9d, 0x9e, 0x11, 0xb2, 0xe2, - 0x4d, 0x98, 0x62, 0x56, 0x69, 0xd1, 0x93, 0xb3, 0xb4, 0xa3, 0x7a, 0xb6, 0xd4, 0x62, 0x67, 0x4b, - 0x1d, 0x26, 0x7a, 0x87, 0x27, 0x9e, 0xd5, 0x32, 0x3b, 0x42, 0x8b, 0xa0, 0x8d, 0xdf, 0x01, 0xa4, - 0x0a, 0x1b, 0x65, 0xba, 0xb8, 0x02, 0xa5, 0x75, 0xd3, 0x3b, 0x14, 0x2a, 0xe1, 0xf7, 0xa1, 0xcc, - 0x9b, 0x23, 0xd9, 0x10, 0xc1, 0xf8, 0xa1, 0xe9, 0x1d, 0x32, 0xc5, 0x2b, 0x06, 0xfb, 0xc6, 0x53, - 0x30, 0xb9, 0x6b, 0x9b, 0x3d, 0xef, 0xd0, 0x91, 0x59, 0x80, 0xde, 0x1c, 0x6a, 0x21, 0x6d, 0x24, - 0xc4, 0x3b, 0x30, 0xe9, 0x92, 0xae, 0x69, 0xd9, 0x96, 0x7d, 0xd0, 0xdc, 0x3f, 0xf1, 0x89, 0x27, - 0x2e, 0x16, 0xd5, 0x80, 0xfc, 0x88, 0x52, 0xa9, 0x6a, 0xfb, 0x1d, 0x67, 0x5f, 0x84, 0x03, 0xf6, - 0x8d, 0x7f, 0xab, 0x41, 0xf9, 0x3d, 0xd3, 0x6f, 0x49, 0x2b, 0xa0, 0x0d, 0xa8, 0x06, 0x41, 0x80, - 0x51, 0x84, 0x2e, 0xb1, 0x54, 0xc4, 0xc6, 0xc8, 0x23, 0xa7, 0xcc, 0x22, 0x95, 0x96, 0x4a, 0x60, - 0xa2, 0x4c, 0xbb, 0x45, 0x3a, 0x81, 0xa8, 0x4c, 0xba, 0x28, 0xc6, 0xa8, 0x8a, 0x52, 0x09, 0x8f, - 0x26, 0xc3, 0x34, 0xcd, 0xdd, 0xf2, 0x17, 0x19, 0x40, 0xc3, 0x3a, 0x7c, 0xd5, 0x93, 0xcb, 0x2d, - 0xa8, 0x7a, 0xbe, 0xe9, 0xfa, 0xcd, 0xd8, 0xb5, 0xab, 0xc2, 0xa8, 0x41, 0x20, 0xbb, 0x03, 0x93, - 0x3d, 0xd7, 0x39, 0x70, 0x89, 0xe7, 0x35, 0x6d, 0xc7, 0xb7, 0x5e, 0x9c, 0x88, 0xc3, 0x5f, 0x55, - 0x92, 0xb7, 0x19, 0x15, 0x35, 0xa0, 0xf0, 0xc2, 0xea, 0xf8, 0xc4, 0xf5, 0xea, 0xb9, 0xf9, 0xec, - 0x42, 0x75, 0xf9, 0xfe, 0x59, 0x56, 0x5b, 0x7c, 0x9b, 0xf1, 0xef, 0x9d, 0xf4, 0x88, 0x21, 0xc7, - 0xaa, 0x07, 0xaa, 0x7c, 0xe4, 0x40, 0x75, 0x0b, 0x20, 0xe4, 0xa7, 0x51, 0x6b, 0x7b, 0xe7, 0xe9, - 0xb3, 0xbd, 0xda, 0x18, 0x2a, 0xc3, 0xc4, 0xf6, 0xce, 0x5a, 0x63, 0xab, 0x41, 0xe3, 0x1a, 0x5e, - 0x92, 0xb6, 0x51, 0x6d, 0x88, 0x66, 0x61, 0xe2, 0x25, 0xa5, 0xca, 0x7b, 0x69, 0xd6, 0x28, 0xb0, - 0xf6, 0x46, 0x1b, 0xff, 0x38, 0x03, 0x15, 0xb1, 0x0b, 0x46, 0xda, 0x8a, 0x2a, 0x44, 0x26, 0x02, - 0x41, 0x4f, 0x6f, 0x7c, 0x77, 0xb4, 0xc5, 0x21, 0x51, 0x36, 0xa9, 0xbb, 0xf3, 0xc5, 0x26, 0x6d, - 0x61, 0xd6, 0xa0, 0x8d, 0xee, 0x42, 0xad, 0xc5, 0xdd, 0x3d, 0x96, 0x93, 0x8c, 0x49, 0x41, 0x57, - 0x52, 0x52, 0x25, 0xd8, 0x6d, 0xa6, 0x27, 0x72, 0x52, 0xd1, 0x28, 0xcb, 0x8d, 0x44, 0x69, 0xe8, - 0x16, 0xe4, 0xc9, 0x80, 0xd8, 0xbe, 0x57, 0x2f, 0xb1, 0x00, 0x56, 0x91, 0xe7, 0xc4, 0x06, 0xa5, - 0x1a, 0xa2, 0x13, 0xff, 0x17, 0x4c, 0xb1, 0xf3, 0xf8, 0x63, 0xd7, 0xb4, 0xd5, 0x8b, 0xc3, 0xde, - 0xde, 0x96, 0x30, 0x1d, 0xfd, 0x44, 0x55, 0xc8, 0x6c, 0xac, 0x89, 0x89, 0x66, 0x36, 0xd6, 0xf0, - 0x27, 0x1a, 0x20, 0x75, 0xdc, 0x48, 0xb6, 0x8c, 0x09, 0x97, 0xf0, 0xd9, 0x10, 0x7e, 0x06, 0x72, - 0xc4, 0x75, 0x1d, 0x97, 0x59, 0xad, 0x68, 0xf0, 0x06, 0xbe, 0x29, 0x74, 0x30, 0xc8, 0xc0, 0x39, - 0x0a, 0x1c, 0x83, 0x4b, 0xd3, 0x02, 0x55, 0x37, 0x61, 0x3a, 0xc2, 0x35, 0x52, 0x20, 0xbd, 0x03, - 0x17, 0x98, 0xb0, 0x4d, 0x42, 0x7a, 0x2b, 0x1d, 0x6b, 0x90, 0x8a, 0xda, 0x83, 0x8b, 0x71, 0xc6, - 0x6f, 0xd6, 0x46, 0xf8, 0x4d, 0x81, 0xb8, 0x67, 0x75, 0xc9, 0x9e, 0xb3, 0x95, 0xae, 0x1b, 0x8d, - 0x8e, 0x47, 0xe4, 0xc4, 0x13, 0x19, 0x87, 0x7d, 0xe3, 0x5f, 0x6b, 0x70, 0x69, 0x68, 0xf8, 0x37, - 0xbc, 0xaa, 0x73, 0x00, 0x07, 0x74, 0xfb, 0x90, 0x36, 0xed, 0xe0, 0x37, 0x59, 0x85, 0x12, 0xe8, - 0x49, 0x03, 0x4c, 0x59, 0xe8, 0x79, 0x08, 0xf9, 0x27, 0xac, 0x88, 0xa4, 0xcc, 0x6a, 0x5c, 0xce, - 0xca, 0x36, 0xbb, 0xfc, 0x6a, 0x5b, 0x34, 0xd8, 0x37, 0xcb, 0xaf, 0x84, 0xb8, 0xcf, 0x8c, 0x2d, - 0x9e, 0xc7, 0x8b, 0x46, 0xd0, 0xa6, 0xe8, 0xad, 0x8e, 0x45, 0x6c, 0x9f, 0xf5, 0x8e, 0xb3, 0x5e, - 0x85, 0x82, 0x17, 0xa1, 0xc6, 0x91, 0x56, 0xda, 0x6d, 0x25, 0x97, 0x07, 0xf2, 0xb4, 0xa8, 0x3c, - 0xfc, 0x1b, 0x0d, 0xa6, 0x94, 0x01, 0x23, 0xd9, 0xee, 0x15, 0xc8, 0xf3, 0x52, 0x99, 0xc8, 0x23, - 0x33, 0xd1, 0x51, 0x1c, 0xc6, 0x10, 0x3c, 0x68, 0x11, 0x0a, 0xfc, 0x4b, 0x1e, 0x56, 0x92, 0xd9, - 0x25, 0x13, 0xbe, 0x05, 0xd3, 0x82, 0x44, 0xba, 0x4e, 0xd2, 0x36, 0x61, 0x06, 0xc5, 0x1f, 0xc3, - 0x4c, 0x94, 0x6d, 0xa4, 0x29, 0x29, 0x4a, 0x66, 0xce, 0xa3, 0xe4, 0x8a, 0x54, 0xf2, 0x59, 0xaf, - 0xad, 0xa4, 0xbd, 0xf8, 0xaa, 0xab, 0x2b, 0x92, 0x89, 0xad, 0x48, 0x30, 0x01, 0x29, 0xe2, 0x5b, - 0x9d, 0xc0, 0xb4, 0xdc, 0x0e, 0x5b, 0x96, 0x17, 0x1c, 0x86, 0x3e, 0x02, 0xa4, 0x12, 0xbf, 0x6d, - 0x85, 0xd6, 0xc8, 0x0b, 0xd7, 0x3c, 0xe8, 0x92, 0x20, 0xd4, 0xd3, 0x53, 0xa6, 0x4a, 0x1c, 0x29, - 0x38, 0x2e, 0xc1, 0xd4, 0x13, 0x67, 0x40, 0xb6, 0x38, 0x35, 0x74, 0x19, 0x7e, 0xcb, 0x08, 0x96, - 0x2d, 0x68, 0x53, 0x70, 0x75, 0xc0, 0x48, 0xe0, 0x7f, 0xd6, 0xa0, 0xbc, 0xd2, 0x31, 0xdd, 0xae, - 0x04, 0x7e, 0x0b, 0xf2, 0xfc, 0xec, 0x2c, 0xee, 0xa2, 0xb7, 0xa3, 0x62, 0x54, 0x5e, 0xde, 0x58, - 0xe1, 0x27, 0x6d, 0x31, 0x8a, 0x2a, 0x2e, 0xca, 0xd5, 0x6b, 0xb1, 0xf2, 0xf5, 0x1a, 0x7a, 0x00, - 0x39, 0x93, 0x0e, 0x61, 0xd1, 0xac, 0x1a, 0xbf, 0xb5, 0x30, 0x69, 0xec, 0x9c, 0xc3, 0xb9, 0xf0, - 0x1b, 0x50, 0x52, 0x10, 0xe8, 0x65, 0xec, 0x71, 0x43, 0x9c, 0x65, 0x56, 0x56, 0xf7, 0x36, 0x9e, - 0xf3, 0x3b, 0x5a, 0x15, 0x60, 0xad, 0x11, 0xb4, 0x33, 0xf8, 0x7d, 0x31, 0x4a, 0xc4, 0x3b, 0x55, - 0x1f, 0x2d, 0x4d, 0x9f, 0xcc, 0xb9, 0xf4, 0x39, 0x86, 0x8a, 0x98, 0xfe, 0x48, 0x1b, 0xf0, 0x35, - 0xc8, 0x33, 0x79, 0x72, 0xff, 0xcd, 0x26, 0xc0, 0xca, 0x50, 0xc5, 0x19, 0xf1, 0x24, 0x54, 0x76, - 0x7d, 0xd3, 0xef, 0x7b, 0x72, 0xff, 0xfd, 0x49, 0x83, 0xaa, 0xa4, 0x8c, 0x5a, 0x33, 0x93, 0xd7, - 0x7d, 0x9e, 0x01, 0x82, 0xcb, 0xfe, 0x45, 0xc8, 0xb7, 0xf7, 0x77, 0xad, 0x8f, 0x64, 0x7d, 0x53, - 0xb4, 0x28, 0xbd, 0xc3, 0x71, 0xf8, 0x23, 0x83, 0x68, 0xd1, 0xbb, 0xa1, 0x6b, 0xbe, 0xf0, 0x37, - 0xec, 0x36, 0x39, 0x66, 0x47, 0xb0, 0x71, 0x23, 0x24, 0xb0, 0xeb, 0x9c, 0x78, 0x8c, 0x60, 0xe7, - 0x2e, 0xf5, 0x71, 0x62, 0x1a, 0xa6, 0x56, 0xfa, 0xfe, 0x61, 0xc3, 0x36, 0xf7, 0x3b, 0x32, 0x62, - 0xe1, 0x19, 0x40, 0x94, 0xb8, 0x66, 0x79, 0x2a, 0xb5, 0x01, 0xd3, 0x94, 0x4a, 0x6c, 0xdf, 0x6a, - 0x29, 0xe1, 0x4d, 0x26, 0x31, 0x2d, 0x96, 0xc4, 0x4c, 0xcf, 0x7b, 0xe9, 0xb8, 0x6d, 0x31, 0xb5, - 0xa0, 0x8d, 0xd7, 0xb8, 0xf0, 0x67, 0x5e, 0x24, 0x4d, 0x7d, 0x55, 0x29, 0x0b, 0xa1, 0x94, 0xc7, - 0xc4, 0x3f, 0x45, 0x0a, 0xbe, 0x0f, 0x17, 0x24, 0xa7, 0xa8, 0x27, 0x9d, 0xc2, 0xbc, 0x03, 0x57, - 0x25, 0xf3, 0xea, 0x21, 0xbd, 0x88, 0x3c, 0x15, 0x80, 0x5f, 0x57, 0xcf, 0x47, 0x50, 0x0f, 0xf4, - 0x64, 0xe7, 0x4e, 0xa7, 0xa3, 0x2a, 0xd0, 0xf7, 0xc4, 0x9e, 0x29, 0x1a, 0xec, 0x9b, 0xd2, 0x5c, - 0xa7, 0x13, 0x1c, 0x09, 0xe8, 0x37, 0x5e, 0x85, 0x59, 0x29, 0x43, 0x9c, 0x08, 0xa3, 0x42, 0x86, - 0x14, 0x4a, 0x12, 0x22, 0x0c, 0x46, 0x87, 0x9e, 0x6e, 0x76, 0x95, 0x33, 0x6a, 0x5a, 0x26, 0x53, - 0x53, 0x64, 0x5e, 0xe0, 0x3b, 0x82, 0x2a, 0xa6, 0x66, 0x0c, 0x41, 0xa6, 0x02, 0x54, 0xb2, 0x58, - 0x08, 0x4a, 0x1e, 0x5a, 0x88, 0x21, 0xd1, 0x1f, 0xc0, 0x5c, 0xa0, 0x04, 0xb5, 0xdb, 0x53, 0xe2, - 0x76, 0x2d, 0xcf, 0x53, 0x8a, 0x14, 0x49, 0x13, 0xbf, 0x0d, 0xe3, 0x3d, 0x22, 0x62, 0x4a, 0x69, - 0x19, 0x2d, 0xf2, 0x27, 0xc3, 0x45, 0x65, 0x30, 0xeb, 0xc7, 0x6d, 0xb8, 0x26, 0xa5, 0x73, 0x8b, - 0x26, 0x8a, 0x8f, 0x2b, 0x25, 0x2f, 0xb0, 0xdc, 0xac, 0xc3, 0x17, 0xd8, 0x2c, 0x5f, 0xfb, 0xa0, - 0x5a, 0xf6, 0x0e, 0x37, 0xa4, 0xf4, 0xad, 0x91, 0x72, 0xc5, 0x26, 0xb7, 0x69, 0xe0, 0x92, 0x23, - 0x09, 0xdb, 0x87, 0x99, 0xa8, 0x27, 0x8f, 0x14, 0xc6, 0x66, 0x20, 0xe7, 0x3b, 0x47, 0x44, 0x06, - 0x31, 0xde, 0x90, 0x0a, 0x07, 0x6e, 0x3e, 0x92, 0xc2, 0x66, 0x28, 0x8c, 0x6d, 0xc9, 0x51, 0xf5, - 0xa5, 0xab, 0x29, 0x0f, 0x5f, 0xbc, 0x81, 0xb7, 0xe1, 0x62, 0x3c, 0x4c, 0x8c, 0xa4, 0xf2, 0x73, - 0xbe, 0x81, 0x93, 0x22, 0xc9, 0x48, 0x72, 0xdf, 0x0d, 0x83, 0x81, 0x12, 0x50, 0x46, 0x12, 0x69, - 0x80, 0x9e, 0x14, 0x5f, 0xfe, 0x13, 0xfb, 0x35, 0x08, 0x37, 0x23, 0x09, 0xf3, 0x42, 0x61, 0xa3, - 0x2f, 0x7f, 0x18, 0x23, 0xb2, 0xa7, 0xc6, 0x08, 0xe1, 0x24, 0x61, 0x14, 0xfb, 0x06, 0x36, 0x9d, - 0xc0, 0x08, 0x03, 0xe8, 0xa8, 0x18, 0x34, 0x87, 0x04, 0x18, 0xac, 0x21, 0x37, 0xb6, 0x1a, 0x76, - 0x47, 0x5a, 0x8c, 0xf7, 0xc2, 0xd8, 0x39, 0x14, 0x99, 0x47, 0x12, 0xfc, 0x3e, 0xcc, 0xa7, 0x07, - 0xe5, 0x51, 0x24, 0xdf, 0xc3, 0x50, 0x0c, 0x0e, 0x94, 0xca, 0x73, 0x7b, 0x09, 0x0a, 0xdb, 0x3b, - 0xbb, 0x4f, 0x57, 0x56, 0x1b, 0x35, 0x6d, 0xf9, 0x9f, 0x59, 0xc8, 0x6c, 0x3e, 0x47, 0xff, 0x0f, - 0x39, 0xfe, 0xde, 0x74, 0xca, 0x23, 0xa3, 0x7e, 0xda, 0x93, 0x1a, 0xbe, 0xf2, 0xc9, 0x5f, 0xff, - 0xf1, 0x79, 0xe6, 0x22, 0x9e, 0x5a, 0x1a, 0xbc, 0x6e, 0x76, 0x7a, 0x87, 0xe6, 0xd2, 0xd1, 0x60, - 0x89, 0xe5, 0x84, 0x87, 0xda, 0x3d, 0xf4, 0x1c, 0xb2, 0x4f, 0xfb, 0x3e, 0x4a, 0x7d, 0x81, 0xd4, - 0xd3, 0x9f, 0xda, 0xb0, 0xce, 0x24, 0xcf, 0xe0, 0x49, 0x55, 0x72, 0xaf, 0xef, 0x53, 0xb9, 0x03, - 0x28, 0xa9, 0xaf, 0x65, 0x67, 0xbe, 0x4d, 0xea, 0x67, 0xbf, 0xc4, 0x61, 0xcc, 0xf0, 0xae, 0xe0, - 0x4b, 0x2a, 0x1e, 0x7f, 0xd4, 0x53, 0xe7, 0xb3, 0x77, 0x6c, 0xa3, 0xd4, 0xe7, 0x4b, 0x3d, 0xfd, - 0x85, 0x2e, 0x79, 0x3e, 0xfe, 0xb1, 0x4d, 0xe5, 0x3a, 0xe2, 0x85, 0xae, 0xe5, 0xa3, 0x6b, 0x09, - 0x8f, 0x38, 0xea, 0x73, 0x85, 0x3e, 0x9f, 0xce, 0x20, 0x90, 0xae, 0x33, 0xa4, 0xcb, 0xf8, 0xa2, - 0x8a, 0xd4, 0x0a, 0xf8, 0x1e, 0x6a, 0xf7, 0x96, 0x0f, 0x21, 0xc7, 0x8a, 0xac, 0xa8, 0x29, 0x3f, - 0xf4, 0x84, 0xf2, 0x70, 0xca, 0x0e, 0x88, 0x94, 0x67, 0xf1, 0x2c, 0x43, 0x9b, 0xc6, 0xd5, 0x00, - 0x8d, 0xd5, 0x59, 0x1f, 0x6a, 0xf7, 0x16, 0xb4, 0x57, 0xb5, 0xe5, 0xef, 0x8f, 0x43, 0x8e, 0xd5, - 0xad, 0x50, 0x0f, 0x20, 0xac, 0x48, 0xc6, 0xe7, 0x39, 0x54, 0xe3, 0x8c, 0xcf, 0x73, 0xb8, 0x98, - 0x89, 0xaf, 0x31, 0xe4, 0x59, 0x3c, 0x13, 0x20, 0xb3, 0x1f, 0x33, 0x2c, 0xb1, 0x0a, 0x15, 0x35, - 0xeb, 0x4b, 0x28, 0x29, 0x95, 0x45, 0x94, 0x24, 0x31, 0x52, 0x9a, 0x8c, 0x6f, 0x93, 0x84, 0xb2, - 0x24, 0xbe, 0xc1, 0x40, 0xaf, 0xe2, 0xba, 0x6a, 0x5c, 0x8e, 0xeb, 0x32, 0x4e, 0x0a, 0xfc, 0xa9, - 0x06, 0xd5, 0x68, 0x75, 0x11, 0xdd, 0x48, 0x10, 0x1d, 0x2f, 0x52, 0xea, 0x37, 0x4f, 0x67, 0x4a, - 0x55, 0x81, 0xe3, 0x1f, 0x11, 0xd2, 0x33, 0x29, 0xa7, 0xb0, 0x3d, 0xfa, 0x81, 0x06, 0x93, 0xb1, - 0x9a, 0x21, 0x4a, 0x82, 0x18, 0xaa, 0x48, 0xea, 0xb7, 0xce, 0xe0, 0x12, 0x9a, 0xdc, 0x61, 0x9a, - 0x5c, 0xc7, 0x57, 0x86, 0x8d, 0xe1, 0x5b, 0x5d, 0xe2, 0x3b, 0x42, 0x9b, 0xe5, 0x7f, 0x65, 0xa1, - 0xb0, 0xca, 0x7f, 0x79, 0x86, 0x7c, 0x28, 0x06, 0x65, 0x38, 0x34, 0x97, 0x54, 0x12, 0x09, 0x8f, - 0xec, 0xfa, 0xb5, 0xd4, 0x7e, 0xa1, 0xc2, 0x6d, 0xa6, 0xc2, 0x3c, 0xbe, 0x1c, 0xa8, 0x20, 0x7e, - 0xe1, 0xb6, 0xc4, 0x2f, 0xdf, 0x4b, 0x66, 0xbb, 0x4d, 0x97, 0xe4, 0x7b, 0x1a, 0x94, 0xd5, 0x6a, - 0x19, 0xba, 0x9e, 0x58, 0x8c, 0x51, 0x0b, 0x6e, 0x3a, 0x3e, 0x8d, 0x45, 0xe0, 0xdf, 0x65, 0xf8, - 0x37, 0xf0, 0x5c, 0x1a, 0xbe, 0xcb, 0xf8, 0xa3, 0x2a, 0xf0, 0x7a, 0x57, 0xb2, 0x0a, 0x91, 0x72, - 0x5a, 0xb2, 0x0a, 0xd1, 0x72, 0xd9, 0xd9, 0x2a, 0xf4, 0x19, 0x3f, 0x55, 0xe1, 0x18, 0x20, 0x2c, - 0x6f, 0xa1, 0x44, 0xe3, 0x2a, 0x97, 0x98, 0xb8, 0x0f, 0x0e, 0x57, 0xc6, 0x12, 0x76, 0x40, 0x0c, - 0xbb, 0x63, 0x79, 0xd4, 0x17, 0x97, 0x7f, 0x9f, 0x83, 0xd2, 0x13, 0xd3, 0xb2, 0x7d, 0x62, 0x9b, - 0x76, 0x8b, 0xa0, 0x03, 0xc8, 0xb1, 0x2c, 0x15, 0x0f, 0x3c, 0x6a, 0xd9, 0x27, 0x1e, 0x78, 0x22, - 0x35, 0x11, 0x7c, 0x8b, 0x41, 0x5f, 0xc3, 0x7a, 0x00, 0xdd, 0x0d, 0xe5, 0x2f, 0xb1, 0x7a, 0x06, - 0x9d, 0xf2, 0x11, 0xe4, 0x79, 0xfd, 0x02, 0xc5, 0xa4, 0x45, 0xea, 0x1c, 0xfa, 0x95, 0xe4, 0xce, - 0xd4, 0x5d, 0xa6, 0x62, 0x79, 0x8c, 0x99, 0x82, 0x7d, 0x07, 0x20, 0xac, 0xd6, 0xc5, 0xed, 0x3b, - 0x54, 0xdc, 0xd3, 0xe7, 0xd3, 0x19, 0x04, 0xf0, 0x3d, 0x06, 0x7c, 0x13, 0x5f, 0x4b, 0x04, 0x6e, - 0x07, 0x03, 0x28, 0x78, 0x0b, 0xc6, 0xd7, 0x4d, 0xef, 0x10, 0xc5, 0x92, 0x90, 0xf2, 0xb0, 0xac, - 0xeb, 0x49, 0x5d, 0x02, 0xea, 0x26, 0x83, 0x9a, 0xc3, 0xb3, 0x89, 0x50, 0x87, 0xa6, 0x47, 0x63, - 0x3a, 0xea, 0xc3, 0x84, 0x7c, 0x2c, 0x46, 0x57, 0x63, 0x36, 0x8b, 0x3e, 0x2c, 0xeb, 0x73, 0x69, - 0xdd, 0x02, 0x70, 0x81, 0x01, 0x62, 0x7c, 0x35, 0xd9, 0xa8, 0x82, 0xfd, 0xa1, 0x76, 0xef, 0x55, - 0x8d, 0x46, 0x54, 0x08, 0x4b, 0x91, 0x43, 0x3b, 0x37, 0x5e, 0xd5, 0x1c, 0xda, 0xb9, 0x43, 0x55, - 0x4c, 0xfc, 0x3a, 0x43, 0x7f, 0x80, 0x17, 0x12, 0xd1, 0x7d, 0xd7, 0xb4, 0xbd, 0x17, 0xc4, 0x7d, - 0xc0, 0x6b, 0x4e, 0xde, 0xa1, 0xd5, 0xa3, 0xbb, 0xf8, 0x47, 0x35, 0x18, 0xa7, 0xc7, 0x36, 0x9a, - 0xcc, 0xc2, 0xdb, 0x6e, 0x5c, 0x9d, 0xa1, 0x1a, 0x53, 0x5c, 0x9d, 0xe1, 0x8b, 0x72, 0x42, 0x32, - 0x63, 0x3f, 0x03, 0x26, 0x8c, 0x8b, 0x1a, 0xde, 0x87, 0x92, 0x72, 0x27, 0x46, 0x09, 0x12, 0xa3, - 0x15, 0xac, 0x78, 0x32, 0x4b, 0xb8, 0x50, 0xe3, 0x79, 0x06, 0xaa, 0xe3, 0x0b, 0x51, 0xd0, 0x36, - 0x67, 0xa3, 0xa8, 0x1f, 0x43, 0x59, 0xbd, 0x3c, 0xa3, 0x04, 0xa1, 0xb1, 0x12, 0x59, 0x3c, 0x64, - 0x25, 0xdd, 0xbd, 0x13, 0x7c, 0x37, 0xf8, 0xd1, 0xb3, 0xe4, 0xa5, 0xe8, 0x1f, 0x42, 0x41, 0x5c, - 0xa9, 0x93, 0xe6, 0x1b, 0x2d, 0xaa, 0x25, 0xcd, 0x37, 0x76, 0x1f, 0x4f, 0x38, 0x19, 0x31, 0x58, - 0x7a, 0x75, 0x90, 0x79, 0x42, 0x40, 0x3e, 0x26, 0x7e, 0x1a, 0x64, 0x58, 0x26, 0x4a, 0x83, 0x54, - 0xae, 0x6d, 0xa7, 0x42, 0x1e, 0x10, 0x5f, 0xb8, 0x94, 0xbc, 0x13, 0xa1, 0x14, 0x89, 0x6a, 0x50, - 0xc6, 0xa7, 0xb1, 0xa4, 0x1e, 0x66, 0x43, 0x54, 0x11, 0x91, 0xd1, 0x77, 0x01, 0xc2, 0xfb, 0x7f, - 0xfc, 0x7c, 0x92, 0x58, 0x44, 0x8c, 0x9f, 0x4f, 0x92, 0x4b, 0x08, 0x09, 0x81, 0x24, 0x04, 0xe7, - 0x07, 0x6a, 0x0a, 0xff, 0x53, 0x0d, 0xd0, 0x70, 0xbd, 0x00, 0xdd, 0x4f, 0x86, 0x48, 0xac, 0x4f, - 0xea, 0xaf, 0x9c, 0x8f, 0x39, 0x35, 0x88, 0x87, 0x7a, 0xb5, 0xd8, 0x90, 0xde, 0x4b, 0xaa, 0xd9, - 0x67, 0x1a, 0x54, 0x22, 0x15, 0x07, 0x74, 0x3b, 0x65, 0x9d, 0x63, 0x35, 0x4e, 0xfd, 0xce, 0x99, - 0x7c, 0xa9, 0x47, 0x38, 0x65, 0x57, 0xc8, 0xe3, 0xeb, 0x0f, 0x35, 0xa8, 0x46, 0xcb, 0x14, 0x28, - 0x05, 0x60, 0xa8, 0x50, 0xaa, 0x2f, 0x9c, 0xcd, 0x78, 0x8e, 0xd5, 0x0a, 0x4f, 0xb4, 0x1f, 0x42, - 0x41, 0x54, 0x37, 0x92, 0xdc, 0x22, 0x5a, 0x67, 0x4d, 0x72, 0x8b, 0x58, 0x69, 0x24, 0xcd, 0x2d, - 0x5c, 0xa7, 0x43, 0x14, 0x4f, 0x14, 0x35, 0x90, 0x34, 0xc8, 0xd3, 0x3d, 0x31, 0x56, 0x40, 0x39, - 0x15, 0x32, 0xf4, 0x44, 0x59, 0x01, 0x41, 0x29, 0x12, 0xcf, 0xf0, 0xc4, 0x78, 0x01, 0x25, 0xcd, - 0x13, 0x19, 0xaa, 0xe2, 0x89, 0x61, 0xc1, 0x22, 0xc9, 0x13, 0x87, 0xaa, 0xc8, 0x49, 0x9e, 0x38, - 0x5c, 0xf3, 0x48, 0x5b, 0x5b, 0x06, 0x1e, 0xf1, 0xc4, 0xe9, 0x84, 0x02, 0x07, 0x7a, 0x25, 0xc5, - 0xa6, 0x89, 0x15, 0x6a, 0xfd, 0xc1, 0x39, 0xb9, 0x4f, 0xf7, 0x00, 0xbe, 0x1a, 0xd2, 0x03, 0x7e, - 0xa9, 0xc1, 0x4c, 0x52, 0x85, 0x04, 0xa5, 0x80, 0xa5, 0x94, 0xb7, 0xf5, 0xc5, 0xf3, 0xb2, 0x9f, - 0xc3, 0x6e, 0x81, 0x4f, 0x3c, 0xaa, 0xfd, 0xf1, 0xcb, 0x39, 0xed, 0x2f, 0x5f, 0xce, 0x69, 0x7f, - 0xfb, 0x72, 0x4e, 0xfb, 0xd9, 0xdf, 0xe7, 0xc6, 0xf6, 0xf3, 0xec, 0xff, 0xe2, 0xbc, 0xfe, 0xef, - 0x00, 0x00, 0x00, 0xff, 0xff, 0x2b, 0x2a, 0x70, 0xa1, 0x12, 0x34, 0x00, 0x00, + // 3580 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0xdd, 0x6f, 0x1b, 0xc7, + 0xb5, 0xd7, 0x92, 0x22, 0x29, 0x1e, 0x7e, 0x88, 0x1a, 0xc9, 0x36, 0xb5, 0xb6, 0x65, 0x79, 0xfc, + 0x25, 0xdb, 0xb1, 0x94, 0x28, 0xb9, 0xf7, 0xc1, 0x37, 0x08, 0xae, 0x2c, 0x31, 0x96, 0x22, 0x59, + 0x72, 0x56, 0xb2, 0x93, 0x0b, 0x04, 0x97, 0x58, 0x91, 0x63, 0x69, 0x21, 0x72, 0x97, 0xd9, 0x5d, + 0xd2, 0x52, 0x9a, 0x02, 0x45, 0x9a, 0xa0, 0x68, 0x1f, 0x9b, 0x87, 0x7e, 0x3d, 0x16, 0x45, 0x91, + 0x3f, 0xa0, 0xe8, 0x1f, 0x50, 0xa0, 0x28, 0xfa, 0xd2, 0x02, 0x7d, 0xea, 0x5b, 0x91, 0xf6, 0xdf, + 0x28, 0x5a, 0xcc, 0xd7, 0xee, 0xec, 0x72, 0x57, 0x52, 0xc2, 0x26, 0x2f, 0xd6, 0xce, 0x99, 0x33, + 0xe7, 0x77, 0xe6, 0xcc, 0x9c, 0x73, 0x66, 0xce, 0xd0, 0x50, 0x74, 0x7b, 0xad, 0xc5, 0x9e, 0xeb, + 0xf8, 0x0e, 0x2a, 0x13, 0xbf, 0xd5, 0xf6, 0x88, 0x3b, 0x20, 0x6e, 0x6f, 0x5f, 0x9f, 0x39, 0x70, + 0x0e, 0x1c, 0xd6, 0xb1, 0x44, 0xbf, 0x38, 0x8f, 0x3e, 0x4b, 0x79, 0x96, 0xba, 0x83, 0x56, 0x8b, + 0xfd, 0xd3, 0xdb, 0x5f, 0x3a, 0x1a, 0x88, 0xae, 0xcb, 0xac, 0xcb, 0xec, 0xfb, 0x87, 0xec, 0x9f, + 0xde, 0x3e, 0xfb, 0x23, 0x3a, 0xaf, 0x1c, 0x38, 0xce, 0x41, 0x87, 0x2c, 0x99, 0x3d, 0x6b, 0xc9, + 0xb4, 0x6d, 0xc7, 0x37, 0x7d, 0xcb, 0xb1, 0x3d, 0xde, 0x8b, 0x3f, 0xd3, 0xa0, 0x6a, 0x10, 0xaf, + 0xe7, 0xd8, 0x1e, 0x59, 0x27, 0x66, 0x9b, 0xb8, 0xe8, 0x2a, 0x40, 0xab, 0xd3, 0xf7, 0x7c, 0xe2, + 0x36, 0xad, 0x76, 0x5d, 0x9b, 0xd7, 0x16, 0xc6, 0x8d, 0xa2, 0xa0, 0x6c, 0xb4, 0xd1, 0x65, 0x28, + 0x76, 0x49, 0x77, 0x9f, 0xf7, 0x66, 0x58, 0xef, 0x04, 0x27, 0x6c, 0xb4, 0x91, 0x0e, 0x13, 0x2e, + 0x19, 0x58, 0x9e, 0xe5, 0xd8, 0xf5, 0xec, 0xbc, 0xb6, 0x90, 0x35, 0x82, 0x36, 0x1d, 0xe8, 0x9a, + 0x2f, 0xfc, 0xa6, 0x4f, 0xdc, 0x6e, 0x7d, 0x9c, 0x0f, 0xa4, 0x84, 0x3d, 0xe2, 0x76, 0xf1, 0xa7, + 0x39, 0x28, 0x1b, 0xa6, 0x7d, 0x40, 0x0c, 0xf2, 0x61, 0x9f, 0x78, 0x3e, 0xaa, 0x41, 0xf6, 0x88, + 0x9c, 0x30, 0xf8, 0xb2, 0x41, 0x3f, 0xf9, 0x78, 0xfb, 0x80, 0x34, 0x89, 0xcd, 0x81, 0xcb, 0x74, + 0xbc, 0x7d, 0x40, 0x1a, 0x76, 0x1b, 0xcd, 0x40, 0xae, 0x63, 0x75, 0x2d, 0x5f, 0xa0, 0xf2, 0x46, + 0x44, 0x9d, 0xf1, 0x98, 0x3a, 0xab, 0x00, 0x9e, 0xe3, 0xfa, 0x4d, 0xc7, 0x6d, 0x13, 0xb7, 0x9e, + 0x9b, 0xd7, 0x16, 0xaa, 0xcb, 0x37, 0x17, 0xd5, 0x85, 0x58, 0x54, 0x15, 0x5a, 0xdc, 0x75, 0x5c, + 0x7f, 0x87, 0xf2, 0x1a, 0x45, 0x4f, 0x7e, 0xa2, 0xb7, 0xa1, 0xc4, 0x84, 0xf8, 0xa6, 0x7b, 0x40, + 0xfc, 0x7a, 0x9e, 0x49, 0xb9, 0x75, 0x86, 0x94, 0x3d, 0xc6, 0x6c, 0x30, 0x78, 0xfe, 0x8d, 0x30, + 0x94, 0x3d, 0xe2, 0x5a, 0x66, 0xc7, 0xfa, 0xc8, 0xdc, 0xef, 0x90, 0x7a, 0x61, 0x5e, 0x5b, 0x98, + 0x30, 0x22, 0x34, 0x3a, 0xff, 0x23, 0x72, 0xe2, 0x35, 0x1d, 0xbb, 0x73, 0x52, 0x9f, 0x60, 0x0c, + 0x13, 0x94, 0xb0, 0x63, 0x77, 0x4e, 0xd8, 0xa2, 0x39, 0x7d, 0xdb, 0xe7, 0xbd, 0x45, 0xd6, 0x5b, + 0x64, 0x14, 0xd6, 0xbd, 0x00, 0xb5, 0xae, 0x65, 0x37, 0xbb, 0x4e, 0xbb, 0x19, 0x18, 0x04, 0x98, + 0x41, 0xaa, 0x5d, 0xcb, 0x7e, 0xe2, 0xb4, 0x0d, 0x69, 0x16, 0xca, 0x69, 0x1e, 0x47, 0x39, 0x4b, + 0x82, 0xd3, 0x3c, 0x56, 0x39, 0x17, 0x61, 0x9a, 0xca, 0x6c, 0xb9, 0xc4, 0xf4, 0x49, 0xc8, 0x5c, + 0x66, 0xcc, 0x53, 0x5d, 0xcb, 0x5e, 0x65, 0x3d, 0x11, 0x7e, 0xf3, 0x78, 0x88, 0xbf, 0x22, 0xf8, + 0xcd, 0xe3, 0x28, 0x3f, 0x5e, 0x84, 0x62, 0x60, 0x73, 0x34, 0x01, 0xe3, 0xdb, 0x3b, 0xdb, 0x8d, + 0xda, 0x18, 0x02, 0xc8, 0xaf, 0xec, 0xae, 0x36, 0xb6, 0xd7, 0x6a, 0x1a, 0x2a, 0x41, 0x61, 0xad, + 0xc1, 0x1b, 0x19, 0xfc, 0x08, 0x20, 0xb4, 0x2e, 0x2a, 0x40, 0x76, 0xb3, 0xf1, 0x7f, 0xb5, 0x31, + 0xca, 0xf3, 0xbc, 0x61, 0xec, 0x6e, 0xec, 0x6c, 0xd7, 0x34, 0x3a, 0x78, 0xd5, 0x68, 0xac, 0xec, + 0x35, 0x6a, 0x19, 0xca, 0xf1, 0x64, 0x67, 0xad, 0x96, 0x45, 0x45, 0xc8, 0x3d, 0x5f, 0xd9, 0x7a, + 0xd6, 0xa8, 0x8d, 0xe3, 0xcf, 0x35, 0xa8, 0x88, 0xf5, 0xe2, 0x3e, 0x81, 0xde, 0x80, 0xfc, 0x21, + 0xf3, 0x0b, 0xb6, 0x15, 0x4b, 0xcb, 0x57, 0x62, 0x8b, 0x1b, 0xf1, 0x1d, 0x43, 0xf0, 0x22, 0x0c, + 0xd9, 0xa3, 0x81, 0x57, 0xcf, 0xcc, 0x67, 0x17, 0x4a, 0xcb, 0xb5, 0x45, 0xee, 0xb0, 0x8b, 0x9b, + 0xe4, 0xe4, 0xb9, 0xd9, 0xe9, 0x13, 0x83, 0x76, 0x22, 0x04, 0xe3, 0x5d, 0xc7, 0x25, 0x6c, 0xc7, + 0x4e, 0x18, 0xec, 0x9b, 0x6e, 0x63, 0xb6, 0x68, 0x62, 0xb7, 0xf2, 0x06, 0xfe, 0x42, 0x03, 0x78, + 0xda, 0xf7, 0xd3, 0x5d, 0x63, 0x06, 0x72, 0x03, 0x2a, 0x58, 0xb8, 0x05, 0x6f, 0x30, 0x9f, 0x20, + 0xa6, 0x47, 0x02, 0x9f, 0xa0, 0x0d, 0x74, 0x09, 0x0a, 0x3d, 0x97, 0x0c, 0x9a, 0x47, 0x03, 0x06, + 0x32, 0x61, 0xe4, 0x69, 0x73, 0x73, 0x80, 0xae, 0x43, 0xd9, 0x3a, 0xb0, 0x1d, 0x97, 0x34, 0xb9, + 0xac, 0x1c, 0xeb, 0x2d, 0x71, 0x1a, 0xd3, 0x5b, 0x61, 0xe1, 0x82, 0xf3, 0x2a, 0xcb, 0x16, 0x25, + 0x61, 0x1b, 0x4a, 0x4c, 0xd5, 0x91, 0xcc, 0x77, 0x37, 0xd4, 0x31, 0xc3, 0x86, 0x0d, 0x9b, 0x50, + 0x68, 0x8d, 0x3f, 0x00, 0xb4, 0x46, 0x3a, 0xc4, 0x27, 0xa3, 0x44, 0x0f, 0xc5, 0x26, 0x59, 0xd5, + 0x26, 0xf8, 0xc7, 0x1a, 0x4c, 0x47, 0xc4, 0x8f, 0x34, 0xad, 0x3a, 0x14, 0xda, 0x4c, 0x18, 0xd7, + 0x20, 0x6b, 0xc8, 0x26, 0xba, 0x0f, 0x13, 0x42, 0x01, 0xaf, 0x9e, 0x4d, 0xd9, 0x34, 0x05, 0xae, + 0x93, 0x87, 0xbf, 0xc8, 0x40, 0x51, 0x4c, 0x74, 0xa7, 0x87, 0x56, 0xa0, 0xe2, 0xf2, 0x46, 0x93, + 0xcd, 0x47, 0x68, 0xa4, 0xa7, 0x07, 0xa1, 0xf5, 0x31, 0xa3, 0x2c, 0x86, 0x30, 0x32, 0xfa, 0x1f, + 0x28, 0x49, 0x11, 0xbd, 0xbe, 0x2f, 0x4c, 0x5e, 0x8f, 0x0a, 0x08, 0xf7, 0xdf, 0xfa, 0x98, 0x01, + 0x82, 0xfd, 0x69, 0xdf, 0x47, 0x7b, 0x30, 0x23, 0x07, 0xf3, 0xd9, 0x08, 0x35, 0xb2, 0x4c, 0xca, + 0x7c, 0x54, 0xca, 0xf0, 0x52, 0xad, 0x8f, 0x19, 0x48, 0x8c, 0x57, 0x3a, 0x55, 0x95, 0xfc, 0x63, + 0x1e, 0xbc, 0x87, 0x54, 0xda, 0x3b, 0xb6, 0x87, 0x55, 0xda, 0x3b, 0xb6, 0x1f, 0x15, 0xa1, 0x20, + 0x5a, 0xf8, 0xb7, 0x19, 0x00, 0xb9, 0x1a, 0x3b, 0x3d, 0xb4, 0x06, 0x55, 0x57, 0xb4, 0x22, 0xd6, + 0xba, 0x9c, 0x68, 0x2d, 0xb1, 0x88, 0x63, 0x46, 0x45, 0x0e, 0xe2, 0xca, 0xbd, 0x05, 0xe5, 0x40, + 0x4a, 0x68, 0xb0, 0xd9, 0x04, 0x83, 0x05, 0x12, 0x4a, 0x72, 0x00, 0x35, 0xd9, 0x7b, 0x70, 0x21, + 0x18, 0x9f, 0x60, 0xb3, 0xeb, 0xa7, 0xd8, 0x2c, 0x10, 0x38, 0x2d, 0x25, 0xa8, 0x56, 0x53, 0x15, + 0x0b, 0xcd, 0x36, 0x9b, 0x60, 0xb6, 0x61, 0xc5, 0xa8, 0xe1, 0x80, 0xe6, 0x4b, 0xde, 0xc4, 0xbf, + 0xcf, 0x42, 0x61, 0xd5, 0xe9, 0xf6, 0x4c, 0x97, 0xae, 0x46, 0xde, 0x25, 0x5e, 0xbf, 0xe3, 0x33, + 0x73, 0x55, 0x97, 0x6f, 0x44, 0x25, 0x0a, 0x36, 0xf9, 0xd7, 0x60, 0xac, 0x86, 0x18, 0x42, 0x07, + 0x8b, 0xf4, 0x98, 0x39, 0xc7, 0x60, 0x91, 0x1c, 0xc5, 0x10, 0xe9, 0xc8, 0xd9, 0xd0, 0x91, 0x75, + 0x28, 0x0c, 0x88, 0x1b, 0xa6, 0xf4, 0xf5, 0x31, 0x43, 0x12, 0xd0, 0x5d, 0x98, 0x8c, 0xa7, 0x97, + 0x9c, 0xe0, 0xa9, 0xb6, 0xa2, 0xd9, 0xe8, 0x06, 0x94, 0x23, 0x39, 0x2e, 0x2f, 0xf8, 0x4a, 0x5d, + 0x25, 0xc5, 0x5d, 0x94, 0x71, 0x95, 0xe6, 0xe3, 0xf2, 0xfa, 0x98, 0x8c, 0xac, 0x91, 0x60, 0x32, + 0x11, 0x0d, 0x26, 0xf8, 0x7f, 0xa1, 0x12, 0x31, 0x04, 0xcd, 0x2f, 0x8d, 0x77, 0x9f, 0xad, 0x6c, + 0xf1, 0x64, 0xf4, 0x98, 0xe5, 0x1f, 0xa3, 0xa6, 0xd1, 0x9c, 0xb6, 0xd5, 0xd8, 0xdd, 0xad, 0x65, + 0x50, 0x05, 0x8a, 0xdb, 0x3b, 0x7b, 0x4d, 0xce, 0x95, 0xc5, 0x6f, 0x06, 0x12, 0x44, 0x32, 0x53, + 0x72, 0xd8, 0x98, 0x92, 0xc3, 0x34, 0x99, 0xc3, 0x32, 0x61, 0x0e, 0xcb, 0x3e, 0xaa, 0x42, 0x99, + 0x1b, 0xaf, 0xd9, 0xb7, 0x69, 0x1e, 0xfd, 0xa5, 0x06, 0x10, 0xba, 0x0a, 0x5a, 0x82, 0x42, 0x8b, + 0x0b, 0xaf, 0x6b, 0x2c, 0xd2, 0x5c, 0x48, 0x5c, 0x0f, 0x43, 0x72, 0xa1, 0xd7, 0xa0, 0xe0, 0xf5, + 0x5b, 0x2d, 0xe2, 0xc9, 0x7c, 0x76, 0x29, 0x1e, 0xec, 0x44, 0x28, 0x32, 0x24, 0x1f, 0x1d, 0xf2, + 0xc2, 0xb4, 0x3a, 0x7d, 0x96, 0xdd, 0x4e, 0x1f, 0x22, 0xf8, 0xf0, 0xcf, 0x34, 0x28, 0x29, 0x3b, + 0xf3, 0x6b, 0x46, 0xd8, 0x2b, 0x50, 0x64, 0x3a, 0x90, 0xb6, 0x88, 0xb1, 0x13, 0x46, 0x48, 0x40, + 0xff, 0x0d, 0x45, 0xb9, 0xbd, 0x65, 0x98, 0xad, 0x27, 0x8b, 0xdd, 0xe9, 0x19, 0x21, 0x2b, 0xde, + 0x84, 0x29, 0x66, 0x95, 0x16, 0x3d, 0x39, 0x4b, 0x3b, 0xaa, 0x67, 0x4b, 0x2d, 0x76, 0xb6, 0xd4, + 0x61, 0xa2, 0x77, 0x78, 0xe2, 0x59, 0x2d, 0xb3, 0x23, 0xb4, 0x08, 0xda, 0xf8, 0x1d, 0x40, 0xaa, + 0xb0, 0x51, 0xa6, 0x8b, 0x2b, 0x50, 0x5a, 0x37, 0xbd, 0x43, 0xa1, 0x12, 0x7e, 0x1f, 0xca, 0xbc, + 0x39, 0x92, 0x0d, 0x11, 0x8c, 0x1f, 0x9a, 0xde, 0x21, 0x53, 0xbc, 0x62, 0xb0, 0x6f, 0x3c, 0x05, + 0x93, 0xbb, 0xb6, 0xd9, 0xf3, 0x0e, 0x1d, 0x99, 0x05, 0xe8, 0xcd, 0xa1, 0x16, 0xd2, 0x46, 0x42, + 0xbc, 0x03, 0x93, 0x2e, 0xe9, 0x9a, 0x96, 0x6d, 0xd9, 0x07, 0xcd, 0xfd, 0x13, 0x9f, 0x78, 0xe2, + 0x62, 0x51, 0x0d, 0xc8, 0x8f, 0x28, 0x95, 0xaa, 0xb6, 0xdf, 0x71, 0xf6, 0x45, 0x38, 0x60, 0xdf, + 0xf8, 0x37, 0x1a, 0x94, 0xdf, 0x33, 0xfd, 0x96, 0xb4, 0x02, 0xda, 0x80, 0x6a, 0x10, 0x04, 0x18, + 0x45, 0xe8, 0x12, 0x4b, 0x45, 0x6c, 0x8c, 0x3c, 0x72, 0xca, 0x2c, 0x52, 0x69, 0xa9, 0x04, 0x26, + 0xca, 0xb4, 0x5b, 0xa4, 0x13, 0x88, 0xca, 0xa4, 0x8b, 0x62, 0x8c, 0xaa, 0x28, 0x95, 0xf0, 0x68, + 0x32, 0x4c, 0xd3, 0xdc, 0x2d, 0x7f, 0x9e, 0x01, 0x34, 0xac, 0xc3, 0x57, 0x3d, 0xb9, 0xdc, 0x82, + 0xaa, 0xe7, 0x9b, 0xae, 0xdf, 0x8c, 0x5d, 0xbb, 0x2a, 0x8c, 0x1a, 0x04, 0xb2, 0x3b, 0x30, 0xd9, + 0x73, 0x9d, 0x03, 0x97, 0x78, 0x5e, 0xd3, 0x76, 0x7c, 0xeb, 0xc5, 0x89, 0x38, 0xfc, 0x55, 0x25, + 0x79, 0x9b, 0x51, 0x51, 0x03, 0x0a, 0x2f, 0xac, 0x8e, 0x4f, 0x5c, 0xaf, 0x9e, 0x9b, 0xcf, 0x2e, + 0x54, 0x97, 0xef, 0x9f, 0x65, 0xb5, 0xc5, 0xb7, 0x19, 0xff, 0xde, 0x49, 0x8f, 0x18, 0x72, 0xac, + 0x7a, 0xa0, 0xca, 0x47, 0x0e, 0x54, 0xb7, 0x00, 0x42, 0x7e, 0x1a, 0xb5, 0xb6, 0x77, 0x9e, 0x3e, + 0xdb, 0xab, 0x8d, 0xa1, 0x32, 0x4c, 0x6c, 0xef, 0xac, 0x35, 0xb6, 0x1a, 0x34, 0xae, 0xe1, 0x25, + 0x69, 0x1b, 0xd5, 0x86, 0x68, 0x16, 0x26, 0x5e, 0x52, 0xaa, 0xbc, 0x97, 0x66, 0x8d, 0x02, 0x6b, + 0x6f, 0xb4, 0xf1, 0x5f, 0x33, 0x50, 0x11, 0xbb, 0x60, 0xa4, 0xad, 0xa8, 0x42, 0x64, 0x22, 0x10, + 0xf4, 0xf4, 0xc6, 0x77, 0x47, 0x5b, 0x1c, 0x12, 0x65, 0x93, 0xba, 0x3b, 0x5f, 0x6c, 0xd2, 0x16, + 0x66, 0x0d, 0xda, 0xe8, 0x2e, 0xd4, 0x5a, 0xdc, 0xdd, 0x63, 0x39, 0xc9, 0x98, 0x14, 0x74, 0x25, + 0x25, 0x55, 0x82, 0xdd, 0x66, 0x7a, 0x22, 0x27, 0x15, 0x8d, 0xb2, 0xdc, 0x48, 0x94, 0x46, 0x17, + 0xfc, 0x85, 0x6b, 0x1e, 0x74, 0x89, 0xed, 0x37, 0xf9, 0x55, 0xa1, 0xc0, 0x17, 0x5c, 0x52, 0x57, + 0x29, 0x91, 0xc9, 0xea, 0xbb, 0x6e, 0x53, 0x52, 0x59, 0x96, 0xca, 0x1a, 0x65, 0x4a, 0x7c, 0x5b, + 0xd0, 0xd0, 0x2d, 0xc8, 0x93, 0x01, 0xb1, 0x7d, 0xaf, 0x5e, 0x62, 0xc1, 0xb0, 0x22, 0xcf, 0x9c, + 0x0d, 0x4a, 0x35, 0x44, 0x27, 0xfe, 0x2f, 0x98, 0x62, 0x67, 0xfb, 0xc7, 0xae, 0x69, 0xab, 0x97, + 0x90, 0xbd, 0xbd, 0x2d, 0xb1, 0x0c, 0xf4, 0x13, 0x55, 0x21, 0xb3, 0xb1, 0x26, 0x8c, 0x96, 0xd9, + 0x58, 0xc3, 0x9f, 0x68, 0x80, 0xd4, 0x71, 0x23, 0xad, 0x4b, 0x4c, 0xb8, 0x84, 0xcf, 0x86, 0xf0, + 0x33, 0x90, 0x23, 0xae, 0xeb, 0xb8, 0x6c, 0x05, 0x8a, 0x06, 0x6f, 0xe0, 0x9b, 0x42, 0x07, 0x83, + 0x0c, 0x9c, 0xa3, 0xc0, 0xc9, 0xb8, 0x34, 0x2d, 0x50, 0x75, 0x13, 0xa6, 0x23, 0x5c, 0x23, 0x05, + 0xe5, 0x3b, 0x70, 0x81, 0x09, 0xdb, 0x24, 0xa4, 0xb7, 0xd2, 0xb1, 0x06, 0xa9, 0xa8, 0x3d, 0xb8, + 0x18, 0x67, 0xfc, 0x66, 0x6d, 0x84, 0xdf, 0x14, 0x88, 0x7b, 0x56, 0x97, 0xec, 0x39, 0x5b, 0xe9, + 0xba, 0xd1, 0x48, 0x7b, 0x44, 0x4e, 0x3c, 0x91, 0xbd, 0xd8, 0x37, 0xfe, 0x95, 0x06, 0x97, 0x86, + 0x86, 0x7f, 0xc3, 0xab, 0x3a, 0x07, 0x70, 0x40, 0xb7, 0x0f, 0x69, 0xd3, 0x0e, 0x7e, 0x2b, 0x56, + 0x28, 0x81, 0x9e, 0x34, 0x58, 0x95, 0x85, 0x9e, 0x87, 0x90, 0x7f, 0xc2, 0x0a, 0x52, 0xca, 0xac, + 0xc6, 0xe5, 0xac, 0x6c, 0xb3, 0xcb, 0xaf, 0xc9, 0x45, 0x83, 0x7d, 0xb3, 0x5c, 0x4d, 0x88, 0xfb, + 0xcc, 0xd8, 0xe2, 0x67, 0x82, 0xa2, 0x11, 0xb4, 0x29, 0x7a, 0xab, 0x63, 0x11, 0xdb, 0x67, 0xbd, + 0xe3, 0xac, 0x57, 0xa1, 0xe0, 0x45, 0xa8, 0x71, 0xa4, 0x95, 0x76, 0x5b, 0x39, 0x17, 0x04, 0xf2, + 0xb4, 0xa8, 0x3c, 0xfc, 0x6b, 0x0d, 0xa6, 0x94, 0x01, 0x23, 0xd9, 0xee, 0x15, 0xc8, 0xf3, 0xb2, + 0x9b, 0xc8, 0x49, 0x33, 0xd1, 0x51, 0x1c, 0xc6, 0x10, 0x3c, 0x68, 0x11, 0x0a, 0xfc, 0x4b, 0x1e, + 0x7c, 0x92, 0xd9, 0x25, 0x13, 0xbe, 0x05, 0xd3, 0x82, 0x44, 0xba, 0x4e, 0xd2, 0x36, 0x61, 0x06, + 0xc5, 0x1f, 0xc3, 0x4c, 0x94, 0x6d, 0xa4, 0x29, 0x29, 0x4a, 0x66, 0xce, 0xa3, 0xe4, 0x8a, 0x54, + 0xf2, 0x59, 0xaf, 0xad, 0xa4, 0xd0, 0xf8, 0xaa, 0xab, 0x2b, 0x92, 0x89, 0xad, 0x48, 0x30, 0x01, + 0x29, 0xe2, 0x5b, 0x9d, 0xc0, 0xb4, 0xdc, 0x0e, 0x5b, 0x96, 0x17, 0x1c, 0xac, 0x3e, 0x02, 0xa4, + 0x12, 0xbf, 0x6d, 0x85, 0xd6, 0x88, 0xcc, 0x19, 0x52, 0xa1, 0x77, 0x00, 0xa9, 0xc4, 0x91, 0x82, + 0xe3, 0x12, 0x4c, 0x3d, 0x71, 0x06, 0x64, 0x8b, 0x53, 0x43, 0x97, 0xe1, 0x37, 0x96, 0x60, 0xd9, + 0x82, 0x36, 0x05, 0x57, 0x07, 0x8c, 0x04, 0xfe, 0x27, 0x0d, 0xca, 0x2b, 0x1d, 0xd3, 0xed, 0x4a, + 0xe0, 0xb7, 0x20, 0xcf, 0xcf, 0xe1, 0xe2, 0x5e, 0x7b, 0x3b, 0x2a, 0x46, 0xe5, 0xe5, 0x8d, 0x15, + 0x7e, 0x6a, 0x17, 0xa3, 0xa8, 0xe2, 0xa2, 0xf4, 0xbd, 0x16, 0x2b, 0x85, 0xaf, 0xa1, 0x07, 0x90, + 0x33, 0xe9, 0x10, 0x16, 0xcd, 0xaa, 0xf1, 0x1b, 0x10, 0x93, 0xc6, 0xce, 0x4c, 0x9c, 0x0b, 0xbf, + 0x01, 0x25, 0x05, 0x81, 0x5e, 0xec, 0x1e, 0x37, 0xc4, 0xb9, 0x68, 0x65, 0x75, 0x6f, 0xe3, 0x39, + 0xbf, 0xef, 0x55, 0x01, 0xd6, 0x1a, 0x41, 0x3b, 0x83, 0xdf, 0x17, 0xa3, 0x44, 0xbc, 0x53, 0xf5, + 0xd1, 0xd2, 0xf4, 0xc9, 0x9c, 0x4b, 0x9f, 0x63, 0xa8, 0x88, 0xe9, 0x8f, 0xb4, 0x01, 0x5f, 0x83, + 0x3c, 0x93, 0x27, 0xf7, 0xdf, 0x6c, 0x02, 0xac, 0x0c, 0x55, 0x9c, 0x11, 0x4f, 0x42, 0x65, 0xd7, + 0x37, 0xfd, 0xbe, 0x27, 0xf7, 0xdf, 0x1f, 0x35, 0xa8, 0x4a, 0xca, 0xa8, 0xf5, 0x37, 0x59, 0x3a, + 0xe0, 0x19, 0x20, 0x28, 0x1c, 0x5c, 0x84, 0x7c, 0x7b, 0x7f, 0xd7, 0xfa, 0x48, 0xd6, 0x4a, 0x45, + 0x8b, 0xd2, 0x3b, 0x1c, 0x87, 0x3f, 0x58, 0x88, 0x16, 0xbd, 0x67, 0xba, 0xe6, 0x0b, 0x7f, 0xc3, + 0x6e, 0x93, 0x63, 0x76, 0x9c, 0x1b, 0x37, 0x42, 0x02, 0xbb, 0x1a, 0x8a, 0x87, 0x0d, 0x76, 0x86, + 0x53, 0x1f, 0x3a, 0xa6, 0x61, 0x6a, 0xa5, 0xef, 0x1f, 0x36, 0x6c, 0x73, 0xbf, 0x23, 0x23, 0x16, + 0x9e, 0x01, 0x44, 0x89, 0x6b, 0x96, 0xa7, 0x52, 0x1b, 0x30, 0x4d, 0xa9, 0xc4, 0xf6, 0xad, 0x96, + 0x12, 0xde, 0x64, 0x12, 0xd3, 0x62, 0x49, 0xcc, 0xf4, 0xbc, 0x97, 0x8e, 0xdb, 0x16, 0x53, 0x0b, + 0xda, 0x78, 0x8d, 0x0b, 0x7f, 0xe6, 0x45, 0xd2, 0xd4, 0x57, 0x95, 0xb2, 0x10, 0x4a, 0x79, 0x4c, + 0xfc, 0x53, 0xa4, 0xe0, 0xfb, 0x70, 0x41, 0x72, 0x8a, 0xda, 0xd4, 0x29, 0xcc, 0x3b, 0x70, 0x55, + 0x32, 0xaf, 0x1e, 0xd2, 0x4b, 0xcd, 0x53, 0x01, 0xf8, 0x75, 0xf5, 0x7c, 0x04, 0xf5, 0x40, 0x4f, + 0x76, 0xee, 0x74, 0x3a, 0xaa, 0x02, 0x7d, 0x4f, 0xec, 0x99, 0xa2, 0xc1, 0xbe, 0x29, 0xcd, 0x75, + 0x3a, 0xc1, 0x91, 0x80, 0x7e, 0xe3, 0x55, 0x98, 0x95, 0x32, 0xc4, 0x89, 0x30, 0x2a, 0x64, 0x48, + 0xa1, 0x24, 0x21, 0xc2, 0x60, 0x74, 0xe8, 0xe9, 0x66, 0x57, 0x39, 0xa3, 0xa6, 0x65, 0x32, 0x35, + 0x45, 0xe6, 0x05, 0xbe, 0x23, 0xa8, 0x62, 0x6a, 0xc6, 0x10, 0x64, 0x2a, 0x40, 0x25, 0x8b, 0x85, + 0xa0, 0xe4, 0xa1, 0x85, 0x18, 0x12, 0xfd, 0x01, 0xcc, 0x05, 0x4a, 0x50, 0xbb, 0x3d, 0x25, 0x6e, + 0xd7, 0xf2, 0x3c, 0xa5, 0xe0, 0x91, 0x34, 0xf1, 0xdb, 0x30, 0xde, 0x23, 0x22, 0xa6, 0x94, 0x96, + 0xd1, 0x22, 0x7f, 0x7e, 0x5c, 0x54, 0x06, 0xb3, 0x7e, 0xdc, 0x86, 0x6b, 0x52, 0x3a, 0xb7, 0x68, + 0xa2, 0xf8, 0xb8, 0x52, 0xf2, 0x32, 0xcc, 0xcd, 0x3a, 0x7c, 0x19, 0xce, 0xf2, 0xb5, 0x0f, 0x2a, + 0x6f, 0xef, 0x70, 0x43, 0x4a, 0xdf, 0x1a, 0x29, 0x57, 0x6c, 0x72, 0x9b, 0x06, 0x2e, 0x39, 0x92, + 0xb0, 0x7d, 0x98, 0x89, 0x7a, 0xf2, 0x48, 0x61, 0x6c, 0x06, 0x72, 0xbe, 0x73, 0x44, 0x64, 0x10, + 0xe3, 0x0d, 0xa9, 0x70, 0xe0, 0xe6, 0x23, 0x29, 0x6c, 0x86, 0xc2, 0xd8, 0x96, 0x1c, 0x55, 0x5f, + 0xba, 0x9a, 0xf2, 0xf0, 0xc5, 0x1b, 0x78, 0x1b, 0x2e, 0xc6, 0xc3, 0xc4, 0x48, 0x2a, 0x3f, 0xe7, + 0x1b, 0x38, 0x29, 0x92, 0x8c, 0x24, 0xf7, 0xdd, 0x30, 0x18, 0x28, 0x01, 0x65, 0x24, 0x91, 0x06, + 0xe8, 0x49, 0xf1, 0xe5, 0x3f, 0xb1, 0x5f, 0x83, 0x70, 0x33, 0x92, 0x30, 0x2f, 0x14, 0x36, 0xfa, + 0xf2, 0x87, 0x31, 0x22, 0x7b, 0x6a, 0x8c, 0x10, 0x4e, 0x12, 0x46, 0xb1, 0x6f, 0x60, 0xd3, 0x09, + 0x8c, 0x30, 0x80, 0x8e, 0x8a, 0x41, 0x73, 0x48, 0x80, 0xc1, 0x1a, 0x72, 0x63, 0xab, 0x61, 0x77, + 0xa4, 0xc5, 0x78, 0x2f, 0x8c, 0x9d, 0x43, 0x91, 0x79, 0x24, 0xc1, 0xef, 0xc3, 0x7c, 0x7a, 0x50, + 0x1e, 0x45, 0xf2, 0x3d, 0x0c, 0xc5, 0xe0, 0x40, 0xa9, 0x3c, 0xdd, 0x97, 0xa0, 0xb0, 0xbd, 0xb3, + 0xfb, 0x74, 0x65, 0xb5, 0x51, 0xd3, 0x96, 0xff, 0x99, 0x85, 0xcc, 0xe6, 0x73, 0xf4, 0xff, 0x90, + 0xe3, 0x6f, 0x57, 0xa7, 0x3c, 0x58, 0xea, 0xa7, 0x3d, 0xcf, 0xe1, 0x2b, 0x9f, 0xfc, 0xe5, 0x1f, + 0x9f, 0x67, 0x2e, 0xe2, 0xa9, 0xa5, 0xc1, 0xeb, 0x66, 0xa7, 0x77, 0x68, 0x2e, 0x1d, 0x0d, 0x96, + 0x58, 0x4e, 0x78, 0xa8, 0xdd, 0x43, 0xcf, 0x21, 0xfb, 0xb4, 0xef, 0xa3, 0xd4, 0xd7, 0x4c, 0x3d, + 0xfd, 0xd9, 0x0e, 0xeb, 0x4c, 0xf2, 0x0c, 0x9e, 0x54, 0x25, 0xf7, 0xfa, 0x3e, 0x95, 0x3b, 0x80, + 0x92, 0xfa, 0xf2, 0x76, 0xe6, 0x3b, 0xa7, 0x7e, 0xf6, 0xab, 0x1e, 0xc6, 0x0c, 0xef, 0x0a, 0xbe, + 0xa4, 0xe2, 0xf1, 0x07, 0x42, 0x75, 0x3e, 0x7b, 0xc7, 0x36, 0x4a, 0x7d, 0x0a, 0xd5, 0xd3, 0x5f, + 0xfb, 0x92, 0xe7, 0xe3, 0x1f, 0xdb, 0x54, 0xae, 0x23, 0x5e, 0xfb, 0x5a, 0x3e, 0xba, 0x96, 0xf0, + 0x20, 0xa4, 0x3e, 0x7d, 0xe8, 0xf3, 0xe9, 0x0c, 0x02, 0xe9, 0x3a, 0x43, 0xba, 0x8c, 0x2f, 0xaa, + 0x48, 0xad, 0x80, 0xef, 0xa1, 0x76, 0x6f, 0xf9, 0x10, 0x72, 0xac, 0x60, 0x8b, 0x9a, 0xf2, 0x43, + 0x4f, 0x28, 0x35, 0xa7, 0xec, 0x80, 0x48, 0xa9, 0x17, 0xcf, 0x32, 0xb4, 0x69, 0x5c, 0x0d, 0xd0, + 0x58, 0xcd, 0xf6, 0xa1, 0x76, 0x6f, 0x41, 0x7b, 0x55, 0x5b, 0xfe, 0xfe, 0x38, 0xe4, 0x58, 0xdd, + 0x0a, 0xf5, 0x00, 0xc2, 0x8a, 0x64, 0x7c, 0x9e, 0x43, 0x35, 0xce, 0xf8, 0x3c, 0x87, 0x8b, 0x99, + 0xf8, 0x1a, 0x43, 0x9e, 0xc5, 0x33, 0x01, 0x32, 0xfb, 0x61, 0xc4, 0x12, 0xab, 0x50, 0x51, 0xb3, + 0xbe, 0x84, 0x92, 0x52, 0x59, 0x44, 0x49, 0x12, 0x23, 0xa5, 0xc9, 0xf8, 0x36, 0x49, 0x28, 0x4b, + 0xe2, 0x1b, 0x0c, 0xf4, 0x2a, 0xae, 0xab, 0xc6, 0xe5, 0xb8, 0x2e, 0xe3, 0xa4, 0xc0, 0x9f, 0x6a, + 0x50, 0x8d, 0x56, 0x17, 0xd1, 0x8d, 0x04, 0xd1, 0xf1, 0x22, 0xa5, 0x7e, 0xf3, 0x74, 0xa6, 0x54, + 0x15, 0x38, 0xfe, 0x11, 0x21, 0x3d, 0x93, 0x72, 0x0a, 0xdb, 0xa3, 0x1f, 0x68, 0x30, 0x19, 0xab, + 0x19, 0xa2, 0x24, 0x88, 0xa1, 0x8a, 0xa4, 0x7e, 0xeb, 0x0c, 0x2e, 0xa1, 0xc9, 0x1d, 0xa6, 0xc9, + 0x75, 0x7c, 0x65, 0xd8, 0x18, 0xbe, 0xd5, 0x25, 0xbe, 0x23, 0xb4, 0x59, 0xfe, 0x57, 0x16, 0x0a, + 0xab, 0xfc, 0x57, 0x6c, 0xc8, 0x87, 0x62, 0x50, 0x86, 0x43, 0x73, 0x49, 0x25, 0x91, 0xf0, 0xc8, + 0xae, 0x5f, 0x4b, 0xed, 0x17, 0x2a, 0xdc, 0x66, 0x2a, 0xcc, 0xe3, 0xcb, 0x81, 0x0a, 0xe2, 0xd7, + 0x72, 0x4b, 0xfc, 0xf2, 0xbd, 0x64, 0xb6, 0xdb, 0x74, 0x49, 0xbe, 0xa7, 0x41, 0x59, 0xad, 0x96, + 0xa1, 0xeb, 0x89, 0xc5, 0x18, 0xb5, 0xe0, 0xa6, 0xe3, 0xd3, 0x58, 0x04, 0xfe, 0x5d, 0x86, 0x7f, + 0x03, 0xcf, 0xa5, 0xe1, 0xbb, 0x8c, 0x3f, 0xaa, 0x02, 0xaf, 0x77, 0x25, 0xab, 0x10, 0x29, 0xa7, + 0x25, 0xab, 0x10, 0x2d, 0x97, 0x9d, 0xad, 0x42, 0x9f, 0xf1, 0x53, 0x15, 0x8e, 0x01, 0xc2, 0xf2, + 0x16, 0x4a, 0x34, 0xae, 0x72, 0x89, 0x89, 0xfb, 0xe0, 0x70, 0x65, 0x2c, 0x61, 0x07, 0xc4, 0xb0, + 0x3b, 0x96, 0x47, 0x7d, 0x71, 0xf9, 0x77, 0x39, 0x28, 0x3d, 0x31, 0x2d, 0xdb, 0x27, 0xb6, 0x69, + 0xb7, 0x08, 0x3a, 0x80, 0x1c, 0xcb, 0x52, 0xf1, 0xc0, 0xa3, 0x96, 0x7d, 0xe2, 0x81, 0x27, 0x52, + 0x13, 0xc1, 0xb7, 0x18, 0xf4, 0x35, 0xac, 0x07, 0xd0, 0xdd, 0x50, 0xfe, 0x12, 0xab, 0x67, 0xd0, + 0x29, 0x1f, 0x41, 0x9e, 0xd7, 0x2f, 0x50, 0x4c, 0x5a, 0xa4, 0xce, 0xa1, 0x5f, 0x49, 0xee, 0x4c, + 0xdd, 0x65, 0x2a, 0x96, 0xc7, 0x98, 0x29, 0xd8, 0x77, 0x00, 0xc2, 0x6a, 0x5d, 0xdc, 0xbe, 0x43, + 0xc5, 0x3d, 0x7d, 0x3e, 0x9d, 0x41, 0x00, 0xdf, 0x63, 0xc0, 0x37, 0xf1, 0xb5, 0x44, 0xe0, 0x76, + 0x30, 0x80, 0x82, 0xb7, 0x60, 0x7c, 0xdd, 0xf4, 0x0e, 0x51, 0x2c, 0x09, 0x29, 0x8f, 0xd4, 0xba, + 0x9e, 0xd4, 0x25, 0xa0, 0x6e, 0x32, 0xa8, 0x39, 0x3c, 0x9b, 0x08, 0x75, 0x68, 0x7a, 0x34, 0xa6, + 0xa3, 0x3e, 0x4c, 0xc8, 0x87, 0x67, 0x74, 0x35, 0x66, 0xb3, 0xe8, 0x23, 0xb5, 0x3e, 0x97, 0xd6, + 0x2d, 0x00, 0x17, 0x18, 0x20, 0xc6, 0x57, 0x93, 0x8d, 0x2a, 0xd8, 0x1f, 0x6a, 0xf7, 0x5e, 0xd5, + 0x68, 0x44, 0x85, 0xb0, 0x14, 0x39, 0xb4, 0x73, 0xe3, 0x55, 0xcd, 0xa1, 0x9d, 0x3b, 0x54, 0xc5, + 0xc4, 0xaf, 0x33, 0xf4, 0x07, 0x78, 0x21, 0x11, 0xdd, 0x77, 0x4d, 0xdb, 0x7b, 0x41, 0xdc, 0x07, + 0xbc, 0xe6, 0xe4, 0x1d, 0x5a, 0x3d, 0xba, 0x8b, 0x7f, 0x54, 0x83, 0x71, 0x7a, 0x6c, 0xa3, 0xc9, + 0x2c, 0xbc, 0xed, 0xc6, 0xd5, 0x19, 0xaa, 0x31, 0xc5, 0xd5, 0x19, 0xbe, 0x28, 0x27, 0x24, 0x33, + 0xf6, 0x93, 0x62, 0xc2, 0xb8, 0xa8, 0xe1, 0x7d, 0x28, 0x29, 0x77, 0x62, 0x94, 0x20, 0x31, 0x5a, + 0xc1, 0x8a, 0x27, 0xb3, 0x84, 0x0b, 0x35, 0x9e, 0x67, 0xa0, 0x3a, 0xbe, 0x10, 0x05, 0x6d, 0x73, + 0x36, 0x8a, 0xfa, 0x31, 0x94, 0xd5, 0xcb, 0x33, 0x4a, 0x10, 0x1a, 0x2b, 0x91, 0xc5, 0x43, 0x56, + 0xd2, 0xdd, 0x3b, 0xc1, 0x77, 0x83, 0x1f, 0x50, 0x4b, 0x5e, 0x8a, 0xfe, 0x21, 0x14, 0xc4, 0x95, + 0x3a, 0x69, 0xbe, 0xd1, 0xa2, 0x5a, 0xd2, 0x7c, 0x63, 0xf7, 0xf1, 0x84, 0x93, 0x11, 0x83, 0xa5, + 0x57, 0x07, 0x99, 0x27, 0x04, 0xe4, 0x63, 0xe2, 0xa7, 0x41, 0x86, 0x65, 0xa2, 0x34, 0x48, 0xe5, + 0xda, 0x76, 0x2a, 0xe4, 0x01, 0xf1, 0x85, 0x4b, 0xc9, 0x3b, 0x11, 0x4a, 0x91, 0xa8, 0x06, 0x65, + 0x7c, 0x1a, 0x4b, 0xea, 0x61, 0x36, 0x44, 0x15, 0x11, 0x19, 0x7d, 0x17, 0x20, 0xbc, 0xff, 0xc7, + 0xcf, 0x27, 0x89, 0x45, 0xc4, 0xf8, 0xf9, 0x24, 0xb9, 0x84, 0x90, 0x10, 0x48, 0x42, 0x70, 0x7e, + 0xa0, 0xa6, 0xf0, 0x3f, 0xd1, 0x00, 0x0d, 0xd7, 0x0b, 0xd0, 0xfd, 0x64, 0x88, 0xc4, 0xfa, 0xa4, + 0xfe, 0xca, 0xf9, 0x98, 0x53, 0x83, 0x78, 0xa8, 0x57, 0x8b, 0x0d, 0xe9, 0xbd, 0xa4, 0x9a, 0x7d, + 0xa6, 0x41, 0x25, 0x52, 0x71, 0x40, 0xb7, 0x53, 0xd6, 0x39, 0x56, 0xe3, 0xd4, 0xef, 0x9c, 0xc9, + 0x97, 0x7a, 0x84, 0x53, 0x76, 0x85, 0x3c, 0xbe, 0xfe, 0x50, 0x83, 0x6a, 0xb4, 0x4c, 0x81, 0x52, + 0x00, 0x86, 0x0a, 0xa5, 0xfa, 0xc2, 0xd9, 0x8c, 0xe7, 0x58, 0xad, 0xf0, 0x44, 0xfb, 0x21, 0x14, + 0x44, 0x75, 0x23, 0xc9, 0x2d, 0xa2, 0x75, 0xd6, 0x24, 0xb7, 0x88, 0x95, 0x46, 0xd2, 0xdc, 0xc2, + 0x75, 0x3a, 0x44, 0xf1, 0x44, 0x51, 0x03, 0x49, 0x83, 0x3c, 0xdd, 0x13, 0x63, 0x05, 0x94, 0x53, + 0x21, 0x43, 0x4f, 0x94, 0x15, 0x10, 0x94, 0x22, 0xf1, 0x0c, 0x4f, 0x8c, 0x17, 0x50, 0xd2, 0x3c, + 0x91, 0xa1, 0x2a, 0x9e, 0x18, 0x16, 0x2c, 0x92, 0x3c, 0x71, 0xa8, 0x8a, 0x9c, 0xe4, 0x89, 0xc3, + 0x35, 0x8f, 0xb4, 0xb5, 0x65, 0xe0, 0x11, 0x4f, 0x9c, 0x4e, 0x28, 0x70, 0xa0, 0x57, 0x52, 0x6c, + 0x9a, 0x58, 0xa1, 0xd6, 0x1f, 0x9c, 0x93, 0xfb, 0x74, 0x0f, 0xe0, 0xab, 0x21, 0x3d, 0xe0, 0x17, + 0x1a, 0xcc, 0x24, 0x55, 0x48, 0x50, 0x0a, 0x58, 0x4a, 0x79, 0x5b, 0x5f, 0x3c, 0x2f, 0xfb, 0x39, + 0xec, 0x16, 0xf8, 0xc4, 0xa3, 0xda, 0x1f, 0xbe, 0x9c, 0xd3, 0xfe, 0xfc, 0xe5, 0x9c, 0xf6, 0xb7, + 0x2f, 0xe7, 0xb4, 0x9f, 0xfe, 0x7d, 0x6e, 0x6c, 0x3f, 0xcf, 0xfe, 0x5f, 0xcf, 0xeb, 0xff, 0x0e, + 0x00, 0x00, 0xff, 0xff, 0x77, 0x94, 0x7e, 0xe0, 0x5e, 0x34, 0x00, 0x00, } diff --git a/etcdserver/etcdserverpb/rpc.proto b/etcdserver/etcdserverpb/rpc.proto index 37916c03f16..eed9cdbe094 100644 --- a/etcdserver/etcdserverpb/rpc.proto +++ b/etcdserver/etcdserverpb/rpc.proto @@ -672,6 +672,10 @@ message WatchResponse { string cancel_reason = 6; repeated mvccpb.Event events = 11; + + int64 fragment_count = 7; + + int64 curr_fragment = 8; } message LeaseGrantRequest { diff --git a/proxy/grpcproxy/watch.go b/proxy/grpcproxy/watch.go index b960c94769a..8d1d4c99390 100644 --- a/proxy/grpcproxy/watch.go +++ b/proxy/grpcproxy/watch.go @@ -15,6 +15,8 @@ package grpcproxy import ( + "fmt" + "log" "sync" "golang.org/x/net/context" @@ -212,6 +214,7 @@ func (wps *watchProxyStream) recvLoop() error { filters: v3rpc.FiltersFromRequest(cr), } if !w.wr.valid() { + log.Fatalf("Watcher range is invalid @proxy\n") w.post(&pb.WatchResponse{WatchId: -1, Created: true, Canceled: true}) continue } @@ -234,8 +237,35 @@ func (wps *watchProxyStream) sendLoop() { if !ok { return } - if err := wps.stream.Send(wresp); err != nil { - return + // The grpc package's defaultServerMaxSendMessageSize, defaultServerMaxReceiveMessageSize + // and the maxSendMesgSize are set to lower numbers in order to demonstrate + // the fragmenting + maxEventsPerMsg := 20 + watchRespSent := false + for !watchRespSent { + if maxEventsPerMsg == 0 { + return + } + for _, fragment := range v3rpc.FragmentWatchResponse(maxEventsPerMsg, wresp) { + if err := wps.stream.Send(fragment); err != nil { + // There isn't a reasonable way of deciphering the cause of the send error. + // One solution that comes to mind is comparing the string error message + // to the expected string for a grpc message indicating that the + // message size is too large, but this approach would fail if grpc + // were to change their error message semantics in the future. The + // approach here is to assume that the error is caused by the message + // being too large and incrementally increasing the number of fragments + // until the maxEventsPerMesg becomes 0 and we are therefore sure + // that the error in the send operation was not caused by the message + // size. + fmt.Printf("Was about to size out. Setting max to %d from %d\n %v\n", maxEventsPerMsg/2, maxEventsPerMsg, err) + maxEventsPerMsg /= 2 + watchRespSent = false + break + } + fmt.Printf("proxy - watchid: %d, count: %d, frag: %d, events: %v\n", fragment.WatchId, fragment.FragmentCount, fragment.CurrFragment, fragment.Events) + watchRespSent = true + } } case <-wps.ctx.Done(): return From d63c431cca2339fab5c67cf0bb9ef6bf9d1ede3e Mon Sep 17 00:00:00 2001 From: mangoslicer Date: Wed, 30 Aug 2017 13:15:49 -0400 Subject: [PATCH 2/2] Made fragmenting responses optional and set watch proxy client to send responses in fragments --- clientv3/integration/fragment_test.go | 114 ++++++++++++++++---------- clientv3/op.go | 25 ++++++ clientv3/watch.go | 78 +++++++++++------- embed/config.go | 11 +-- etcdserver/api/v3rpc/grpc.go | 3 +- etcdserver/api/v3rpc/watch.go | 93 +++++++++++---------- etcdserver/config.go | 3 + etcdserver/etcdserverpb/rpc.proto | 8 +- integration/cluster.go | 7 ++ proxy/grpcproxy/watch.go | 43 ++-------- proxy/grpcproxy/watch_broadcast.go | 5 ++ proxy/grpcproxy/watcher.go | 12 +-- 12 files changed, 233 insertions(+), 169 deletions(-) diff --git a/clientv3/integration/fragment_test.go b/clientv3/integration/fragment_test.go index d2e99ee905a..9a6005f08ff 100644 --- a/clientv3/integration/fragment_test.go +++ b/clientv3/integration/fragment_test.go @@ -17,28 +17,22 @@ package integration import ( "context" "fmt" - "log" + "strconv" "testing" - "time" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/integration" + "github.com/coreos/etcd/mvcc/mvccpb" "github.com/coreos/etcd/pkg/testutil" - "github.com/etcd/proxy/grpcproxy" - "github.com/etcd/proxy/grpcproxy/adapter" + "github.com/coreos/etcd/proxy/grpcproxy" + "github.com/coreos/etcd/proxy/grpcproxy/adapter" ) -// TestFragmentationStopsAfterServerFailure tests the edge case where either -// the server of watch proxy fails to send a message due to errors not related to -// the fragment size, such as a member failure. In that case, -// the server or watch proxy should continue to reduce the message size (a default -// action choosen because there is no way of telling whether the error caused -// by the send operation is message size related or caused by some other issue) -// until the message size becomes zero and thus we are certain that the message -// size is not the issue causing the send operation to fail. -func TestFragmentationStopsAfterServerFailure(t *testing.T) { +func TestResponseWithoutFragmenting(t *testing.T) { defer testutil.AfterTest(t) - clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + // MaxResponseBytes will overflow to 1000 once the grpcOverheadBytes, + // which have a value of 512 * 1024, are added to MaxResponseBytes. + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3, MaxResponseBytes: ^uint(0) - (512*1024 - 1 - 1000)}) defer clus.Terminate(t) cfg := clientv3.Config{ @@ -56,30 +50,56 @@ func TestFragmentationStopsAfterServerFailure(t *testing.T) { t.Error(err) } + // Create and register watch proxy. + wp, _ := grpcproxy.NewWatchProxy(clus.Client(0)) + wc := adapter.WatchServerToWatchClient(wp) + w := clientv3.NewWatchFromWatchClient(wc) + kv := clus.Client(0) - for i := 0; i < 25; i++ { + for i := 0; i < 10; i++ { _, err = kv.Put(context.TODO(), fmt.Sprintf("foo%d", i), "bar", clientv3.WithLease(firstLease.ID)) if err != nil { t.Error(err) } } - kv.Watch(context.TODO(), "foo", clientv3.WithRange("z")) + // Does not include the clientv3.WithFragmentedResponse option. + wChannel := w.Watch(context.TODO(), "foo", clientv3.WithRange("z")) _, err = clus.Client(0).Revoke(context.Background(), firstLease.ID) if err != nil { t.Error(err) } - clus.Members[0].Stop(t) - time.Sleep(10 * time.Second) - log.Fatal("Printed the log") + + r, ok := <-wChannel + if !ok { + t.Error() + } + keyDigitSum := 0 + responseSum := 0 + if len(r.Events) != 10 { + t.Errorf("Expected 10 events, got %d\n", len(r.Events)) + } + for i := 0; i < 10; i++ { + if r.Events[i].Type != mvccpb.DELETE { + t.Errorf("Expected DELETE event, got %d", r.Events[i].Type) + } + keyDigitSum += i + digit, err := strconv.Atoi((string(r.Events[i].Kv.Key)[3:])) + if err != nil { + t.Error("Failed to convert %s to int", (string(r.Events[i].Kv.Key)[3:])) + } + responseSum += digit + } + if keyDigitSum != responseSum { + t.Errorf("Expected digits of keys received in the response to sum to %d, but got %d\n", keyDigitSum, responseSum) + } } -// TestFragmentingWithOverlappingWatchers tests that events are fragmented -// on the server and watch proxy and pieced back together on the client-side -// properly. -func TestFragmentingWithOverlappingWatchers(t *testing.T) { +func TestFragmenting(t *testing.T) { defer testutil.AfterTest(t) - clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + // MaxResponseBytes will overflow to 1000 once the grpcOverheadBytes, + // which have a value of 512 * 1024, are added to MaxResponseBytes. + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3, MaxResponseBytes: ^uint(0) - (512*1024 - 1 - 1000)}) defer clus.Terminate(t) cfg := clientv3.Config{ @@ -96,10 +116,6 @@ func TestFragmentingWithOverlappingWatchers(t *testing.T) { if err != nil { t.Error(err) } - secondLease, err := clus.Client(0).Grant(context.Background(), 10000) - if err != nil { - t.Error(err) - } // Create and register watch proxy wp, _ := grpcproxy.NewWatchProxy(clus.Client(0)) @@ -107,31 +123,39 @@ func TestFragmentingWithOverlappingWatchers(t *testing.T) { w := clientv3.NewWatchFromWatchClient(wc) kv := clus.Client(0) - for i := 0; i < 25; i++ { + for i := 0; i < 100; i++ { _, err = kv.Put(context.TODO(), fmt.Sprintf("foo%d", i), "bar", clientv3.WithLease(firstLease.ID)) if err != nil { t.Error(err) } - _, err = kv.Put(context.TODO(), fmt.Sprintf("buzz%d", i), "fizz", clientv3.WithLease(secondLease.ID)) - if err != nil { - t.Error(err) - } } - - w.Watch(context.TODO(), "foo", clientv3.WithRange("z")) - w.Watch(context.TODO(), "buzz", clientv3.WithRange("z ")) - + wChannel := w.Watch(context.TODO(), "foo", clientv3.WithRange("z"), clientv3.WithFragmentedResponse()) _, err = clus.Client(0).Revoke(context.Background(), firstLease.ID) if err != nil { t.Error(err) } - _, err = clus.Client(0).Revoke(context.Background(), secondLease.ID) - if err != nil { - t.Error(err) - } - - // Wait for the revokation process to finish - time.Sleep(10 * time.Second) - log.Fatal("Printed the log") + r, ok := <-wChannel + if !ok { + t.Error() + } + keyDigitSum := 0 + responseSum := 0 + if len(r.Events) != 100 { + t.Errorf("Expected 100 events, got %d\n", len(r.Events)) + } + for i := 0; i < 100; i++ { + if r.Events[i].Type != mvccpb.DELETE { + t.Errorf("Expected DELETE event, got %d", r.Events[i].Type) + } + keyDigitSum += i + digit, err := strconv.Atoi((string(r.Events[i].Kv.Key)[3:])) + if err != nil { + t.Error("Failed to convert %s to int", (string(r.Events[i].Kv.Key)[3:])) + } + responseSum += digit + } + if keyDigitSum != responseSum { + t.Errorf("Expected digits of keys received in the response to sum to %d, but got %d\n", keyDigitSum, responseSum) + } } diff --git a/clientv3/op.go b/clientv3/op.go index ef6f04368a3..6cd8ac2ffa6 100644 --- a/clientv3/op.go +++ b/clientv3/op.go @@ -73,6 +73,15 @@ type Op struct { cmps []Cmp thenOps []Op elseOps []Op + + // fragmentResponse allows watch clients to toggle whether to send + // watch responses that are too large to send over a rpc stream in fragments. + // Sending in fragments is an opt-in feature in order to preserve compatibility + // with older clients that can not handle responses being split into fragments. + fragmentResponse bool + // combineFragments indicates whether watch clients should combine + // fragments or relay the watch response in fragmented form. + combineFragments bool } // accesors / mutators @@ -415,6 +424,22 @@ func WithCreatedNotify() OpOption { } } +// WithFragmentedResponse makes the watch server send watch responses +// that are too large to send over the rpc stream in fragments. +// The fragmenting feature is an opt-in feature in order to maintain +// compatibility with older versions of clients. +func WithFragmentedResponse() OpOption { + return func(op *Op) { op.fragmentResponse = true } +} + +// WithFragments allows watch clients to passively relay their receieved +// fragmented watch responses. +// This option is handy for the watch client's belonging to watch proxies, +// because these watch clients can be set to passively send along fragments +// rather than reassembling and then fragmenting them such that they can be +// sent to the user's watch client. +func WithFragments() OpOption { return func(op *Op) { op.combineFragments = true } } + // WithFilterPut discards PUT events from the watcher. func WithFilterPut() OpOption { return func(op *Op) { op.filterPut = true } diff --git a/clientv3/watch.go b/clientv3/watch.go index 0f8ea25e1a6..4cdcf52a3c8 100644 --- a/clientv3/watch.go +++ b/clientv3/watch.go @@ -68,6 +68,10 @@ type WatchResponse struct { // cancelReason is a reason of canceling watch cancelReason string + + // MoreFragments indicates that more fragments composing one large + // watch fragment are expected. + MoreFragments bool } // IsCreate returns true if the event tells that the key is newly created. @@ -145,6 +149,10 @@ type watchGrpcStream struct { resumec chan struct{} // closeErr is the error that closed the watch stream closeErr error + + // combineFragments indicates whether watch clients should combine + // fragments or relay the watch response in fragmented form. + combineFragments bool } // watchRequest is issued by the subscriber to start a new watcher @@ -157,6 +165,9 @@ type watchRequest struct { createdNotify bool // progressNotify is for progress updates progressNotify bool + // fragmentResponse allows watch clients to toggle whether to send + // watch responses that are too large to send over a rpc stream in fragments. + fragmentResponse bool // filters is the list of events to filter out filters []pb.WatchCreateRequest_FilterType // get the previous key-value pair before the event happens @@ -241,15 +252,16 @@ func (w *watcher) Watch(ctx context.Context, key string, opts ...OpOption) Watch } wr := &watchRequest{ - ctx: ctx, - createdNotify: ow.createdNotify, - key: string(ow.key), - end: string(ow.end), - rev: ow.rev, - progressNotify: ow.progressNotify, - filters: filters, - prevKV: ow.prevKV, - retc: make(chan chan WatchResponse, 1), + ctx: ctx, + createdNotify: ow.createdNotify, + key: string(ow.key), + end: string(ow.end), + rev: ow.rev, + progressNotify: ow.progressNotify, + fragmentResponse: ow.fragmentResponse, + filters: filters, + prevKV: ow.prevKV, + retc: make(chan chan WatchResponse, 1), } ok := false @@ -267,6 +279,7 @@ func (w *watcher) Watch(ctx context.Context, key string, opts ...OpOption) Watch wgs := w.streams[ctxKey] if wgs == nil { wgs = w.newWatcherGrpcStream(ctx) + wgs.combineFragments = ow.combineFragments w.streams[ctxKey] = wgs } donec := wgs.donec @@ -424,7 +437,7 @@ func (w *watchGrpcStream) run() { } cancelSet := make(map[int64]struct{}) - var combinedFragments *pb.WatchResponse + var fragmentsToSend *pb.WatchResponse for { select { // Watch() requested @@ -450,25 +463,26 @@ func (w *watchGrpcStream) run() { } // New events from the watch client case pbresp := <-w.respc: - fmt.Printf("Clientv3/watcher - watchid: %d, count: %d, frag: %d, events: %v\n", pbresp.WatchId, pbresp.FragmentCount, pbresp.CurrFragment, pbresp.Events) - if combinedFragments == nil || (combinedFragments.WatchId != pbresp.WatchId && combinedFragments.CurrFragment+1 != pbresp.CurrFragment) { - combinedFragments = pbresp + if w.combineFragments { + fragmentsToSend = pbresp } else { - combinedFragments.Events = append(combinedFragments.Events, pbresp.Events...) - combinedFragments.CurrFragment = pbresp.CurrFragment - } - if combinedFragments.FragmentCount != combinedFragments.CurrFragment { - fmt.Printf("Waiting for more fragments ...\n") - break + if fragmentsToSend == nil || fragmentsToSend.WatchId != pbresp.WatchId { + fragmentsToSend = pbresp + } else { + fragmentsToSend.Events = append(fragmentsToSend.Events, pbresp.Events...) + fragmentsToSend.MoreFragments = pbresp.MoreFragments + } + if fragmentsToSend.MoreFragments { + break + } } - fmt.Println("All fragments receieved\n") switch { case pbresp.Created: // response to head of queue creation if ws := w.resuming[0]; ws != nil { - w.addSubstream(combinedFragments, ws) - combinedFragments = nil - w.dispatchEvent(pbresp) + w.addSubstream(fragmentsToSend, ws) + w.dispatchEvent(fragmentsToSend) + fragmentsToSend = nil w.resuming[0] = nil } if ws := w.nextResume(); ws != nil { @@ -483,8 +497,8 @@ func (w *watchGrpcStream) run() { } default: // dispatch to appropriate watch stream - if ok := w.dispatchEvent(combinedFragments); ok { - combinedFragments = nil + if ok := w.dispatchEvent(fragmentsToSend); ok { + fragmentsToSend = nil break } // watch response on unexpected watch id; cancel id @@ -551,6 +565,7 @@ func (w *watchGrpcStream) dispatchEvent(pbresp *pb.WatchResponse) bool { Created: pbresp.Created, Canceled: pbresp.Canceled, cancelReason: pbresp.CancelReason, + MoreFragments: pbresp.MoreFragments, } ws, ok := w.substreams[pbresp.WatchId] if !ok { @@ -798,12 +813,13 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error) // toPB converts an internal watch request structure to its protobuf messagefunc (wr *watchRequest) func (wr *watchRequest) toPB() *pb.WatchRequest { req := &pb.WatchCreateRequest{ - StartRevision: wr.rev, - Key: []byte(wr.key), - RangeEnd: []byte(wr.end), - ProgressNotify: wr.progressNotify, - Filters: wr.filters, - PrevKv: wr.prevKV, + StartRevision: wr.rev, + Key: []byte(wr.key), + RangeEnd: []byte(wr.end), + ProgressNotify: wr.progressNotify, + FragmentResponse: wr.fragmentResponse, + Filters: wr.filters, + PrevKv: wr.prevKV, } cr := &pb.WatchRequest_CreateRequest{CreateRequest: req} return &pb.WatchRequest{RequestUnion: cr} diff --git a/embed/config.go b/embed/config.go index 80410312230..0ebc158894e 100644 --- a/embed/config.go +++ b/embed/config.go @@ -37,11 +37,12 @@ const ( ClusterStateFlagNew = "new" ClusterStateFlagExisting = "existing" - DefaultName = "default" - DefaultMaxSnapshots = 5 - DefaultMaxWALs = 5 - DefaultMaxTxnOps = uint(128) - DefaultMaxRequestBytes = 1.5 * 1024 * 1024 + DefaultName = "default" + DefaultMaxSnapshots = 5 + DefaultMaxWALs = 5 + DefaultMaxTxnOps = uint(128) + DefaultMaxRequestBytes = 1.5 * 1024 * 1024 + DefaultMaxResponseBytes = 1.5 * 1024 * 1024 DefaultListenPeerURLs = "http://localhost:2380" DefaultListenClientURLs = "http://localhost:2379" diff --git a/etcdserver/api/v3rpc/grpc.go b/etcdserver/api/v3rpc/grpc.go index 10d58a88028..56aae72eca4 100644 --- a/etcdserver/api/v3rpc/grpc.go +++ b/etcdserver/api/v3rpc/grpc.go @@ -41,7 +41,8 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config) *grpc.Server { } opts = append(opts, grpc.UnaryInterceptor(newUnaryInterceptor(s))) opts = append(opts, grpc.StreamInterceptor(newStreamInterceptor(s))) - opts = append(opts, grpc.MaxMsgSize(int(s.Cfg.MaxRequestBytes+grpcOverheadBytes))) + opts = append(opts, grpc.MaxRecvMsgSize(int(s.Cfg.MaxRequestBytes+grpcOverheadBytes))) + opts = append(opts, grpc.MaxSendMsgSize(int(s.Cfg.MaxResponseBytes+grpcOverheadBytes))) grpcServer := grpc.NewServer(opts...) pb.RegisterKVServer(grpcServer, NewQuotaKVServer(s)) diff --git a/etcdserver/api/v3rpc/watch.go b/etcdserver/api/v3rpc/watch.go index 1c07f517d70..c484f063e60 100644 --- a/etcdserver/api/v3rpc/watch.go +++ b/etcdserver/api/v3rpc/watch.go @@ -15,9 +15,7 @@ package v3rpc import ( - "fmt" "io" - "math" "sync" "time" @@ -99,8 +97,9 @@ type serverWatchStream struct { // progress tracks the watchID that stream might need to send // progress to. // TODO: combine progress and prevKV into a single struct? - progress map[mvcc.WatchID]bool - prevKV map[mvcc.WatchID]bool + progress map[mvcc.WatchID]bool + prevKV map[mvcc.WatchID]bool + fragmentWatchResponse map[mvcc.WatchID]bool // closec indicates the stream is closed. closec chan struct{} @@ -122,10 +121,11 @@ func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) { gRPCStream: stream, watchStream: ws.watchable.NewWatchStream(), // chan for sending control response like watcher created and canceled. - ctrlStream: make(chan *pb.WatchResponse, ctrlStreamBufLen), - progress: make(map[mvcc.WatchID]bool), - prevKV: make(map[mvcc.WatchID]bool), - closec: make(chan struct{}), + ctrlStream: make(chan *pb.WatchResponse, ctrlStreamBufLen), + progress: make(map[mvcc.WatchID]bool), + prevKV: make(map[mvcc.WatchID]bool), + fragmentWatchResponse: make(map[mvcc.WatchID]bool), + closec: make(chan struct{}), ag: ws.ag, } @@ -238,11 +238,14 @@ func (sws *serverWatchStream) recvLoop() error { } sws.mu.Unlock() } + sws.fragmentWatchResponse[id] = creq.FragmentResponse + wr := &pb.WatchResponse{ - Header: sws.newResponseHeader(wsrev), - WatchId: int64(id), - Created: true, - Canceled: id == -1, + Header: sws.newResponseHeader(wsrev), + WatchId: int64(id), + Created: true, + Canceled: id == -1, + MoreFragments: false, } select { case sws.ctrlStream <- wr: @@ -337,35 +340,37 @@ func (sws *serverWatchStream) sendLoop() { } mvcc.ReportEventReceived(len(evs)) - // The grpc package's defaultServerMaxSendMessageSize, defaultServerMaxReceiveMessageSize - // and the maxSendMesgSize are set to lower numbers in order to demonstrate - // the fragmenting. - maxEventsPerMsg := 20 - watchRespSent := false - for !watchRespSent { - // There isn't a reasonable way of deciphering the cause of the send error. - // One solution that comes to mind is comparing the string error message - // to the expected string for a grpc message indicating that the - // message size is too large, but this approach would fail if grpc - // were to change their error message semantics in the future. The - // approach here is to assume that the error is caused by the message - // being too large and incrementally increasing the number of fragments - // until the maxEventsPerMesg becomes 0 and we are therefore sure - // that the error in the send operation was not caused by the message - // size. - if maxEventsPerMsg == 0 { + if !sws.fragmentWatchResponse[wresp.WatchID] { + if err := sws.gRPCStream.Send(wr); err != nil { return } - for _, fragment := range FragmentWatchResponse(maxEventsPerMsg, wr) { + continue + } + var sendFragments func(maxEventsPerMsg int, wr *pb.WatchResponse) error + sendFragments = func(maxEventsPerMsg int, wr *pb.WatchResponse) error { + for _, fragment := range fragmentWatchResponse(maxEventsPerMsg, wr) { if err := sws.gRPCStream.Send(fragment); err != nil { - fmt.Printf("Was about to size out. Setting max to %d from %d\n %v\n", maxEventsPerMsg/2, maxEventsPerMsg, err) - maxEventsPerMsg /= 2 - watchRespSent = false - break + // maxEventsPerMsg should never reach zero. If maxEventsPerMsg is + // about to approach zero, the size of the watch response is probably + // not causing the send operation to fail. + if maxEventsPerMsg/2 == 0 { + return err + } + // further fragment the fragment which caused the send operation + // to fail. + err = sendFragments(maxEventsPerMsg/2, fragment) + if err != nil { + return err + } + continue } - fmt.Printf("etcdserver - watchid: %d, count: %d, frag: %d, events: %v\n", fragment.WatchId, fragment.FragmentCount, fragment.CurrFragment, fragment.Events) - watchRespSent = true } + return nil + } + // Start with trying to send all of the events in the watch response. + err := sendFragments(len(wr.Events), wr) + if err != nil { + return } sws.mu.Lock() @@ -416,10 +421,8 @@ func (sws *serverWatchStream) sendLoop() { } } -func FragmentWatchResponse(maxSendMesgSize int, wr *pb.WatchResponse) []*pb.WatchResponse { +func fragmentWatchResponse(maxSendMesgSize int, wr *pb.WatchResponse) []*pb.WatchResponse { var fragmentWrs []*pb.WatchResponse - totalFragments := int64(math.Ceil(float64(len(wr.Events)) / float64(maxSendMesgSize))) - currFragmentCount := 1 for i := 0; i < len(wr.Events); i += maxSendMesgSize { eventRangeEnd := i + maxSendMesgSize if eventRangeEnd > len(wr.Events) { @@ -430,16 +433,20 @@ func FragmentWatchResponse(maxSendMesgSize int, wr *pb.WatchResponse) []*pb.Watc WatchId: int64(wr.WatchId), Events: wr.Events[i:eventRangeEnd], CompactRevision: wr.CompactRevision, - FragmentCount: int64(totalFragments), - CurrFragment: int64(currFragmentCount), + MoreFragments: true, } - currFragmentCount++ - fmt.Printf("Apending: total: %d, curr: %d\n", wresp.FragmentCount, wresp.CurrFragment) fragmentWrs = append(fragmentWrs, wresp) } if len(fragmentWrs) == 0 { return []*pb.WatchResponse{wr} } + // Since fragmentWatchResponse might be called in order to further fragment + // an existing fragment, we want to make sure to only set MoreFragments of the + // last fragment to false only if fragmentWatchResponse with a watch response + // parameter that itself has MoreFragments set to false. + if !wr.MoreFragments { + fragmentWrs[len(fragmentWrs)-1].MoreFragments = false + } return fragmentWrs } diff --git a/etcdserver/config.go b/etcdserver/config.go index f6ed1f1bae7..7fcf31ac528 100644 --- a/etcdserver/config.go +++ b/etcdserver/config.go @@ -60,6 +60,9 @@ type ServerConfig struct { // MaxRequestBytes is the maximum request size to send over raft. MaxRequestBytes uint + // MaxResponseBytes is the maximum response size to send over raft. + MaxResponseBytes uint + StrictReconfigCheck bool // ClientCertAuthEnabled is true when cert has been signed by the client CA. diff --git a/etcdserver/etcdserverpb/rpc.proto b/etcdserver/etcdserverpb/rpc.proto index eed9cdbe094..3cc72113fbc 100644 --- a/etcdserver/etcdserverpb/rpc.proto +++ b/etcdserver/etcdserverpb/rpc.proto @@ -639,6 +639,10 @@ message WatchCreateRequest { // If prev_kv is set, created watcher gets the previous KV before the event happens. // If the previous KV is already compacted, nothing will be returned. bool prev_kv = 6; + + // sendInFragments allows watch clients to toggle whether to send + // watch responses that are too large to send over a rpc stream in fragments. + bool fragment_response = 7; } message WatchCancelRequest { @@ -673,9 +677,7 @@ message WatchResponse { repeated mvccpb.Event events = 11; - int64 fragment_count = 7; - - int64 curr_fragment = 8; + bool more_fragments = 7; } message LeaseGrantRequest { diff --git a/integration/cluster.go b/integration/cluster.go index ed245eca2f2..5d7e3e88c03 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -97,6 +97,7 @@ type ClusterConfig struct { QuotaBackendBytes int64 MaxTxnOps uint MaxRequestBytes uint + MaxResponseBytes uint } type cluster struct { @@ -230,6 +231,7 @@ func (c *cluster) mustNewMember(t *testing.T) *member { quotaBackendBytes: c.cfg.QuotaBackendBytes, maxTxnOps: c.cfg.MaxTxnOps, maxRequestBytes: c.cfg.MaxRequestBytes, + maxResponseBytes: c.cfg.MaxResponseBytes, }) m.DiscoveryURL = c.cfg.DiscoveryURL if c.cfg.UseGRPC { @@ -498,6 +500,7 @@ type memberConfig struct { quotaBackendBytes int64 maxTxnOps uint maxRequestBytes uint + maxResponseBytes uint } // mustNewMember return an inited member with the given name. If peerTLS is @@ -553,6 +556,10 @@ func mustNewMember(t *testing.T, mcfg memberConfig) *member { if m.MaxRequestBytes == 0 { m.MaxRequestBytes = embed.DefaultMaxRequestBytes } + m.MaxResponseBytes = mcfg.maxResponseBytes + if m.MaxResponseBytes == 0 { + m.MaxResponseBytes = embed.DefaultMaxRequestBytes + } m.AuthToken = "simple" // for the purpose of integration testing, simple token is enough return m } diff --git a/proxy/grpcproxy/watch.go b/proxy/grpcproxy/watch.go index 8d1d4c99390..15253bfdd20 100644 --- a/proxy/grpcproxy/watch.go +++ b/proxy/grpcproxy/watch.go @@ -15,8 +15,6 @@ package grpcproxy import ( - "fmt" - "log" "sync" "golang.org/x/net/context" @@ -208,13 +206,13 @@ func (wps *watchProxyStream) recvLoop() error { id: wps.nextWatcherID, wps: wps, - nextrev: cr.StartRevision, - progress: cr.ProgressNotify, - prevKV: cr.PrevKv, - filters: v3rpc.FiltersFromRequest(cr), + nextrev: cr.StartRevision, + progress: cr.ProgressNotify, + prevKV: cr.PrevKv, + fragmentResponse: cr.FragmentResponse, + filters: v3rpc.FiltersFromRequest(cr), } if !w.wr.valid() { - log.Fatalf("Watcher range is invalid @proxy\n") w.post(&pb.WatchResponse{WatchId: -1, Created: true, Canceled: true}) continue } @@ -237,35 +235,8 @@ func (wps *watchProxyStream) sendLoop() { if !ok { return } - // The grpc package's defaultServerMaxSendMessageSize, defaultServerMaxReceiveMessageSize - // and the maxSendMesgSize are set to lower numbers in order to demonstrate - // the fragmenting - maxEventsPerMsg := 20 - watchRespSent := false - for !watchRespSent { - if maxEventsPerMsg == 0 { - return - } - for _, fragment := range v3rpc.FragmentWatchResponse(maxEventsPerMsg, wresp) { - if err := wps.stream.Send(fragment); err != nil { - // There isn't a reasonable way of deciphering the cause of the send error. - // One solution that comes to mind is comparing the string error message - // to the expected string for a grpc message indicating that the - // message size is too large, but this approach would fail if grpc - // were to change their error message semantics in the future. The - // approach here is to assume that the error is caused by the message - // being too large and incrementally increasing the number of fragments - // until the maxEventsPerMesg becomes 0 and we are therefore sure - // that the error in the send operation was not caused by the message - // size. - fmt.Printf("Was about to size out. Setting max to %d from %d\n %v\n", maxEventsPerMsg/2, maxEventsPerMsg, err) - maxEventsPerMsg /= 2 - watchRespSent = false - break - } - fmt.Printf("proxy - watchid: %d, count: %d, frag: %d, events: %v\n", fragment.WatchId, fragment.FragmentCount, fragment.CurrFragment, fragment.Events) - watchRespSent = true - } + if err := wps.stream.Send(wresp); err != nil { + return } case <-wps.ctx.Done(): return diff --git a/proxy/grpcproxy/watch_broadcast.go b/proxy/grpcproxy/watch_broadcast.go index 5e750bdb0d4..b322159e4f1 100644 --- a/proxy/grpcproxy/watch_broadcast.go +++ b/proxy/grpcproxy/watch_broadcast.go @@ -57,6 +57,11 @@ func newWatchBroadcast(wp *watchProxy, w *watcher, update func(*watchBroadcast)) clientv3.WithRev(wb.nextrev), clientv3.WithPrevKV(), clientv3.WithCreatedNotify(), + clientv3.WithFragments(), + } + + if w.fragmentResponse { + opts = append(opts, clientv3.WithFragmentedResponse()) } wch := wp.cw.Watch(cctx, w.wr.key, opts...) diff --git a/proxy/grpcproxy/watcher.go b/proxy/grpcproxy/watcher.go index 7387caf4dbd..f7f07ce3986 100644 --- a/proxy/grpcproxy/watcher.go +++ b/proxy/grpcproxy/watcher.go @@ -34,10 +34,11 @@ func (wr *watchRange) valid() bool { type watcher struct { // user configuration - wr watchRange - filters []mvcc.FilterFunc - progress bool - prevKV bool + wr watchRange + filters []mvcc.FilterFunc + progress bool + prevKV bool + fragmentResponse bool // id is the id returned to the client on its watch stream. id int64 @@ -97,7 +98,7 @@ func (w *watcher) send(wr clientv3.WatchResponse) { events = append(events, ev) } - if lastRev >= w.nextrev { + if lastRev >= w.nextrev && !wr.MoreFragments { w.nextrev = lastRev + 1 } @@ -113,6 +114,7 @@ func (w *watcher) send(wr clientv3.WatchResponse) { CompactRevision: wr.CompactRevision, WatchId: w.id, Events: events, + MoreFragments: wr.MoreFragments, }) }