lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-04-05 20:52:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

tiered storage: can copy to s3, read from s3

Browse Source 
master not aware tiered volume yet, file assigning is not working yet
This commit is contained in:
Chris Lu 2019-12-02 15:08:28 -08:00
parent 228231f3d7
commit ec8de250e2
13 changed files with 689 additions and 286 deletions

17
weed/pb/volume_server.proto
View File

@ -68,7 +68,7 @@ service VolumeServer {
}
// tiered storage
rpc VolumeTierCopyDatToRemote (VolumeTierCopyDatToRemoteRequest) returns (VolumeTierCopyDatToRemoteResponse) {
rpc VolumeTierCopyDatToRemote (VolumeTierCopyDatToRemoteRequest) returns (stream VolumeTierCopyDatToRemoteResponse) {
}
// query
@ -329,18 +329,27 @@ message MemStatus {
}
// tired storage on volume servers
message TieredVolume {
message RemoteFile {
string backend_type = 1;
string backend_name = 2;
uint64 version = 3;
string backend_id = 2;
string key = 3;
uint64 offset = 4;
uint64 file_size = 5;
uint64 modified_time = 6;
}
message VolumeTierInfo {
repeated RemoteFile files = 1;
}
message VolumeTierCopyDatToRemoteRequest {
uint32 volume_id = 1;
string collection = 2;
string destination_backend_name = 3;
bool keep_local_dat_file = 4;
}
message VolumeTierCopyDatToRemoteResponse {
int64 processed = 1;
float processedPercentage = 2;
}
// select on volume servers

534
weed/pb/volume_server_pb/volume_server.pb.go
View File

@ -65,7 +65,8 @@ It has these top-level messages:
ReadVolumeFileStatusResponse
DiskStatus
MemStatus
TieredVolume
RemoteFile
VolumeTierInfo
VolumeTierCopyDatToRemoteRequest
VolumeTierCopyDatToRemoteResponse
QueryRequest
@ -1398,49 +1399,90 @@ func (m *MemStatus) GetStack() uint64 {
}
// tired storage on volume servers
type TieredVolume struct {
BackendType string `protobuf:"bytes,1,opt,name=backend_type,json=backendType" json:"backend_type,omitempty"`
BackendName string `protobuf:"bytes,2,opt,name=backend_name,json=backendName" json:"backend_name,omitempty"`
Version uint64 `protobuf:"varint,3,opt,name=version" json:"version,omitempty"`
type RemoteFile struct {
BackendType string `protobuf:"bytes,1,opt,name=backend_type,json=backendType" json:"backend_type,omitempty"`
BackendId string `protobuf:"bytes,2,opt,name=backend_id,json=backendId" json:"backend_id,omitempty"`
Key string `protobuf:"bytes,3,opt,name=key" json:"key,omitempty"`
Offset uint64 `protobuf:"varint,4,opt,name=offset" json:"offset,omitempty"`
FileSize uint64 `protobuf:"varint,5,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
ModifiedTime uint64 `protobuf:"varint,6,opt,name=modified_time,json=modifiedTime" json:"modified_time,omitempty"`
}
func (m *TieredVolume) Reset() { *m = TieredVolume{} }
func (m *TieredVolume) String() string { return proto.CompactTextString(m) }
func (*TieredVolume) ProtoMessage() {}
func (*TieredVolume) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56} }
func (m *RemoteFile) Reset() { *m = RemoteFile{} }
func (m *RemoteFile) String() string { return proto.CompactTextString(m) }
func (*RemoteFile) ProtoMessage() {}
func (*RemoteFile) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56} }
func (m *TieredVolume) GetBackendType() string {
func (m *RemoteFile) GetBackendType() string {
if m != nil {
return m.BackendType
}
return ""
}
func (m *TieredVolume) GetBackendName() string {
func (m *RemoteFile) GetBackendId() string {
if m != nil {
return m.BackendName
return m.BackendId
}
return ""
}
func (m *TieredVolume) GetVersion() uint64 {
func (m *RemoteFile) GetKey() string {
if m != nil {
return m.Version
return m.Key
}
return ""
}
func (m *RemoteFile) GetOffset() uint64 {
if m != nil {
return m.Offset
}
return 0
}
func (m *RemoteFile) GetFileSize() uint64 {
if m != nil {
return m.FileSize
}
return 0
}
func (m *RemoteFile) GetModifiedTime() uint64 {
if m != nil {
return m.ModifiedTime
}
return 0
}
type VolumeTierInfo struct {
Files []*RemoteFile `protobuf:"bytes,1,rep,name=files" json:"files,omitempty"`
}
func (m *VolumeTierInfo) Reset() { *m = VolumeTierInfo{} }
func (m *VolumeTierInfo) String() string { return proto.CompactTextString(m) }
func (*VolumeTierInfo) ProtoMessage() {}
func (*VolumeTierInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} }
func (m *VolumeTierInfo) GetFiles() []*RemoteFile {
if m != nil {
return m.Files
}
return nil
}
type VolumeTierCopyDatToRemoteRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
DestinationBackendName string `protobuf:"bytes,3,opt,name=destination_backend_name,json=destinationBackendName" json:"destination_backend_name,omitempty"`
KeepLocalDatFile bool `protobuf:"varint,4,opt,name=keep_local_dat_file,json=keepLocalDatFile" json:"keep_local_dat_file,omitempty"`
}
func (m *VolumeTierCopyDatToRemoteRequest) Reset() { *m = VolumeTierCopyDatToRemoteRequest{} }
func (m *VolumeTierCopyDatToRemoteRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeTierCopyDatToRemoteRequest) ProtoMessage() {}
func (*VolumeTierCopyDatToRemoteRequest) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{57}
return fileDescriptor0, []int{58}
}
func (m *VolumeTierCopyDatToRemoteRequest) GetVolumeId() uint32 {
@ -1464,14 +1506,37 @@ func (m *VolumeTierCopyDatToRemoteRequest) GetDestinationBackendName() string {
return ""
}
func (m *VolumeTierCopyDatToRemoteRequest) GetKeepLocalDatFile() bool {
if m != nil {
return m.KeepLocalDatFile
}
return false
}
type VolumeTierCopyDatToRemoteResponse struct {
Processed int64 `protobuf:"varint,1,opt,name=processed" json:"processed,omitempty"`
ProcessedPercentage float32 `protobuf:"fixed32,2,opt,name=processedPercentage" json:"processedPercentage,omitempty"`
}
func (m *VolumeTierCopyDatToRemoteResponse) Reset() { *m = VolumeTierCopyDatToRemoteResponse{} }
func (m *VolumeTierCopyDatToRemoteResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeTierCopyDatToRemoteResponse) ProtoMessage() {}
func (*VolumeTierCopyDatToRemoteResponse) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{58}
return fileDescriptor0, []int{59}
}
func (m *VolumeTierCopyDatToRemoteResponse) GetProcessed() int64 {
if m != nil {
return m.Processed
}
return 0
}
func (m *VolumeTierCopyDatToRemoteResponse) GetProcessedPercentage() float32 {
if m != nil {
return m.ProcessedPercentage
}
return 0
}
// select on volume servers
@ -1486,7 +1551,7 @@ type QueryRequest struct {
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{59} }
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60} }
func (m *QueryRequest) GetSelections() []string {
if m != nil {
@ -1532,7 +1597,7 @@ type QueryRequest_Filter struct {
func (m *QueryRequest_Filter) Reset() { *m = QueryRequest_Filter{} }
func (m *QueryRequest_Filter) String() string { return proto.CompactTextString(m) }
func (*QueryRequest_Filter) ProtoMessage() {}
func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{59, 0} }
func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60, 0} }
func (m *QueryRequest_Filter) GetField() string {
if m != nil {
@ -1567,7 +1632,7 @@ func (m *QueryRequest_InputSerialization) Reset() { *m = QueryRequest_In
func (m *QueryRequest_InputSerialization) String() string { return proto.CompactTextString(m) }
func (*QueryRequest_InputSerialization) ProtoMessage() {}
func (*QueryRequest_InputSerialization) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 1}
return fileDescriptor0, []int{60, 1}
}
func (m *QueryRequest_InputSerialization) GetCompressionType() string {
@ -1615,7 +1680,7 @@ func (m *QueryRequest_InputSerialization_CSVInput) Reset() {
func (m *QueryRequest_InputSerialization_CSVInput) String() string { return proto.CompactTextString(m) }
func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {}
func (*QueryRequest_InputSerialization_CSVInput) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 1, 0}
return fileDescriptor0, []int{60, 1, 0}
}
func (m *QueryRequest_InputSerialization_CSVInput) GetFileHeaderInfo() string {
@ -1677,7 +1742,7 @@ func (m *QueryRequest_InputSerialization_JSONInput) Reset() {
func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) }
func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {}
func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 1, 1}
return fileDescriptor0, []int{60, 1, 1}
}
func (m *QueryRequest_InputSerialization_JSONInput) GetType() string {
@ -1698,7 +1763,7 @@ func (m *QueryRequest_InputSerialization_ParquetInput) String() string {
}
func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {}
func (*QueryRequest_InputSerialization_ParquetInput) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 1, 2}
return fileDescriptor0, []int{60, 1, 2}
}
type QueryRequest_OutputSerialization struct {
@ -1710,7 +1775,7 @@ func (m *QueryRequest_OutputSerialization) Reset() { *m = QueryRequest_O
func (m *QueryRequest_OutputSerialization) String() string { return proto.CompactTextString(m) }
func (*QueryRequest_OutputSerialization) ProtoMessage() {}
func (*QueryRequest_OutputSerialization) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 2}
return fileDescriptor0, []int{60, 2}
}
func (m *QueryRequest_OutputSerialization) GetCsvOutput() *QueryRequest_OutputSerialization_CSVOutput {
@ -1743,7 +1808,7 @@ func (m *QueryRequest_OutputSerialization_CSVOutput) String() string {
}
func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {}
func (*QueryRequest_OutputSerialization_CSVOutput) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 2, 0}
return fileDescriptor0, []int{60, 2, 0}
}
func (m *QueryRequest_OutputSerialization_CSVOutput) GetQuoteFields() string {
@ -1793,7 +1858,7 @@ func (m *QueryRequest_OutputSerialization_JSONOutput) String() string {
}
func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {}
func (*QueryRequest_OutputSerialization_JSONOutput) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{59, 2, 1}
return fileDescriptor0, []int{60, 2, 1}
}
func (m *QueryRequest_OutputSerialization_JSONOutput) GetRecordDelimiter() string {
@ -1810,7 +1875,7 @@ type QueriedStripe struct {
func (m *QueriedStripe) Reset() { *m = QueriedStripe{} }
func (m *QueriedStripe) String() string { return proto.CompactTextString(m) }
func (*QueriedStripe) ProtoMessage() {}
func (*QueriedStripe) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60} }
func (*QueriedStripe) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{61} }
func (m *QueriedStripe) GetRecords() []byte {
if m != nil {
@ -1876,7 +1941,8 @@ func init() {
proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse")
proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus")
proto.RegisterType((*MemStatus)(nil), "volume_server_pb.MemStatus")
proto.RegisterType((*TieredVolume)(nil), "volume_server_pb.TieredVolume")
proto.RegisterType((*RemoteFile)(nil), "volume_server_pb.RemoteFile")
proto.RegisterType((*VolumeTierInfo)(nil), "volume_server_pb.VolumeTierInfo")
proto.RegisterType((*VolumeTierCopyDatToRemoteRequest)(nil), "volume_server_pb.VolumeTierCopyDatToRemoteRequest")
proto.RegisterType((*VolumeTierCopyDatToRemoteResponse)(nil), "volume_server_pb.VolumeTierCopyDatToRemoteResponse")
proto.RegisterType((*QueryRequest)(nil), "volume_server_pb.QueryRequest")
@ -1932,7 +1998,7 @@ type VolumeServerClient interface {
VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error)
VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error)
// tiered storage
VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (*VolumeTierCopyDatToRemoteResponse, error)
VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierCopyDatToRemoteClient, error)
// query
Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error)
}
@ -2271,17 +2337,40 @@ func (c *volumeServerClient) VolumeEcBlobDelete(ctx context.Context, in *VolumeE
return out, nil
}
func (c *volumeServerClient) VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (*VolumeTierCopyDatToRemoteResponse, error) {
out := new(VolumeTierCopyDatToRemoteResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", in, out, c.cc, opts...)
func (c *volumeServerClient) VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierCopyDatToRemoteClient, error) {
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[4], c.cc, "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", opts...)
if err != nil {
return nil, err
}
return out, nil
x := &volumeServerVolumeTierCopyDatToRemoteClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type VolumeServer_VolumeTierCopyDatToRemoteClient interface {
Recv() (*VolumeTierCopyDatToRemoteResponse, error)
grpc.ClientStream
}
type volumeServerVolumeTierCopyDatToRemoteClient struct {
grpc.ClientStream
}
func (x *volumeServerVolumeTierCopyDatToRemoteClient) Recv() (*VolumeTierCopyDatToRemoteResponse, error) {
m := new(VolumeTierCopyDatToRemoteResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) {
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[4], c.cc, "/volume_server_pb.VolumeServer/Query", opts...)
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[5], c.cc, "/volume_server_pb.VolumeServer/Query", opts...)
if err != nil {
return nil, err
}
@ -2345,7 +2434,7 @@ type VolumeServerServer interface {
VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error
VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error)
// tiered storage
VolumeTierCopyDatToRemote(context.Context, *VolumeTierCopyDatToRemoteRequest) (*VolumeTierCopyDatToRemoteResponse, error)
VolumeTierCopyDatToRemote(*VolumeTierCopyDatToRemoteRequest, VolumeServer_VolumeTierCopyDatToRemoteServer) error
// query
Query(*QueryRequest, VolumeServer_QueryServer) error
}
@ -2834,22 +2923,25 @@ func _VolumeServer_VolumeEcBlobDelete_Handler(srv interface{}, ctx context.Conte
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_VolumeTierCopyDatToRemote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeTierCopyDatToRemoteRequest)
if err := dec(in); err != nil {
return nil, err
func _VolumeServer_VolumeTierCopyDatToRemote_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(VolumeTierCopyDatToRemoteRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
if interceptor == nil {
return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(ctx, req.(*VolumeTierCopyDatToRemoteRequest))
}
return interceptor(ctx, in, info, handler)
return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(m, &volumeServerVolumeTierCopyDatToRemoteServer{stream})
}
type VolumeServer_VolumeTierCopyDatToRemoteServer interface {
Send(*VolumeTierCopyDatToRemoteResponse) error
grpc.ServerStream
}
type volumeServerVolumeTierCopyDatToRemoteServer struct {
grpc.ServerStream
}
func (x *volumeServerVolumeTierCopyDatToRemoteServer) Send(m *VolumeTierCopyDatToRemoteResponse) error {
return x.ServerStream.SendMsg(m)
}
func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error {
@ -2965,10 +3057,6 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
MethodName: "VolumeEcBlobDelete",
Handler: _VolumeServer_VolumeEcBlobDelete_Handler,
},
{
MethodName: "VolumeTierCopyDatToRemote",
Handler: _VolumeServer_VolumeTierCopyDatToRemote_Handler,
},
},
Streams: []grpc.StreamDesc{
{
@ -2991,6 +3079,11 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
Handler: _VolumeServer_VolumeEcShardRead_Handler,
ServerStreams: true,
},
{
StreamName: "VolumeTierCopyDatToRemote",
Handler: _VolumeServer_VolumeTierCopyDatToRemote_Handler,
ServerStreams: true,
},
{
StreamName: "Query",
Handler: _VolumeServer_Query_Handler,
@ -3003,169 +3096,176 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 2613 bytes of a gzipped FileDescriptorProto
// 2735 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x3a, 0xcd, 0x73, 0x1c, 0x47,
0xf5, 0x5a, 0xaf, 0x3e, 0x76, 0xdf, 0xae, 0x2c, 0xb9, 0x25, 0x4b, 0xeb, 0xb1, 0x25, 0xcb, 0xe3,
0x24, 0x96, 0xed, 0x44, 0x76, 0xe4, 0xdf, 0x8f, 0x98, 0x84, 0x00, 0xb6, 0x6c, 0x83, 0x49, 0x2c,
0x93, 0x91, 0x62, 0x02, 0x4e, 0x31, 0xd5, 0x9a, 0x69, 0x59, 0x83, 0x66, 0xa6, 0xc7, 0x33, 0x3d,
0xb2, 0xd7, 0x05, 0xa7, 0x50, 0xdc, 0xf8, 0x03, 0x72, 0xe6, 0xce, 0x81, 0x0b, 0x7f, 0x00, 0x17,
0xfe, 0x00, 0xb8, 0x72, 0xe1, 0xcc, 0x81, 0x1b, 0x55, 0x5c, 0xa8, 0xfe, 0x98, 0xd9, 0xf9, 0xd4,
0x8e, 0x62, 0x57, 0x51, 0xdc, 0x66, 0x5f, 0xbf, 0xef, 0x7e, 0xaf, 0xfb, 0xf5, 0x7b, 0x0b, 0x0b,
0x47, 0xd4, 0x8d, 0x3d, 0x62, 0x46, 0x24, 0x3c, 0x22, 0xe1, 0x46, 0x10, 0x52, 0x46, 0xd1, 0x7c,
0x0e, 0x68, 0x06, 0x7b, 0xfa, 0x0d, 0x40, 0x77, 0x31, 0xb3, 0x0e, 0xee, 0x11, 0x97, 0x30, 0x62,
0x90, 0xe7, 0x31, 0x89, 0x18, 0x3a, 0x07, 0x9d, 0x7d, 0xc7, 0x25, 0xa6, 0x63, 0x47, 0x83, 0xd6,
0x5a, 0x7b, 0xbd, 0x6b, 0xcc, 0xf0, 0xdf, 0x0f, 0xed, 0x48, 0x7f, 0x0c, 0x0b, 0x39, 0x82, 0x28,
0xa0, 0x7e, 0x44, 0xd0, 0x6d, 0x98, 0x09, 0x49, 0x14, 0xbb, 0x4c, 0x12, 0xf4, 0x36, 0x57, 0x37,
0x8a, 0xb2, 0x36, 0x52, 0x92, 0xd8, 0x65, 0x46, 0x82, 0xae, 0x7f, 0xd5, 0x82, 0x7e, 0x76, 0x05,
0x2d, 0xc3, 0x8c, 0x12, 0x3e, 0x68, 0xad, 0xb5, 0xd6, 0xbb, 0xc6, 0xb4, 0x94, 0x8d, 0x96, 0x60,
0x3a, 0x62, 0x98, 0xc5, 0xd1, 0xe0, 0xd4, 0x5a, 0x6b, 0x7d, 0xca, 0x50, 0xbf, 0xd0, 0x22, 0x4c,
0x91, 0x30, 0xa4, 0xe1, 0xa0, 0x2d, 0xd0, 0xe5, 0x0f, 0x84, 0x60, 0x32, 0x72, 0x5e, 0x91, 0xc1,
0xe4, 0x5a, 0x6b, 0x7d, 0xd6, 0x10, 0xdf, 0x68, 0x00, 0x33, 0x47, 0x24, 0x8c, 0x1c, 0xea, 0x0f,
0xa6, 0x04, 0x38, 0xf9, 0xa9, 0xcf, 0xc0, 0xd4, 0x7d, 0x2f, 0x60, 0x43, 0xfd, 0x03, 0x18, 0x3c,
0xc1, 0x56, 0x1c, 0x7b, 0x4f, 0x84, 0xfa, 0x5b, 0x07, 0xc4, 0x3a, 0x4c, 0xdc, 0x72, 0x1e, 0xba,
0xca, 0x28, 0xa5, 0xdb, 0xac, 0xd1, 0x91, 0x80, 0x87, 0xb6, 0xfe, 0x7d, 0x38, 0x57, 0x41, 0xa8,
0xdc, 0x73, 0x19, 0x66, 0x9f, 0xe1, 0x70, 0x0f, 0x3f, 0x23, 0x66, 0x88, 0x99, 0x43, 0x05, 0x75,
0xcb, 0xe8, 0x2b, 0xa0, 0xc1, 0x61, 0xfa, 0x53, 0xd0, 0x72, 0x1c, 0xa8, 0x17, 0x60, 0x8b, 0x35,
0x11, 0x8e, 0xd6, 0xa0, 0x17, 0x84, 0x04, 0xbb, 0x2e, 0xb5, 0x30, 0x23, 0xc2, 0x3f, 0x6d, 0x23,
0x0b, 0xd2, 0x57, 0xe0, 0x7c, 0x25, 0x73, 0xa9, 0xa0, 0x7e, 0xbb, 0xa0, 0x3d, 0xf5, 0x3c, 0xa7,
0x91, 0x68, 0xfd, 0x42, 0x49, 0x6b, 0x41, 0xa9, 0xf8, 0x7e, 0xbb, 0xb0, 0xea, 0x12, 0xec, 0xc7,
0x41, 0x23, 0xc6, 0x45, 0x8d, 0x13, 0xd2, 0x94, 0xf3, 0xb2, 0x0c, 0x9b, 0x2d, 0xea, 0xba, 0xc4,
0x62, 0x0e, 0xf5, 0x13, 0xb6, 0xab, 0x00, 0x56, 0x0a, 0x54, 0x41, 0x94, 0x81, 0xe8, 0x1a, 0x0c,
0xca, 0xa4, 0x8a, 0xed, 0xdf, 0x5a, 0x70, 0xf6, 0x8e, 0x72, 0x9a, 0x14, 0xdc, 0x68, 0x03, 0xf2,
0x22, 0x4f, 0x15, 0x45, 0x16, 0x37, 0xa8, 0x5d, 0xda, 0x20, 0x8e, 0x11, 0x92, 0xc0, 0x75, 0x2c,
0x2c, 0x58, 0x4c, 0x0a, 0x16, 0x59, 0x10, 0x9a, 0x87, 0x36, 0x63, 0xae, 0x88, 0xdc, 0xae, 0xc1,
0x3f, 0xd1, 0x26, 0x2c, 0x79, 0xc4, 0xa3, 0xe1, 0xd0, 0xf4, 0x70, 0x60, 0x7a, 0xf8, 0xa5, 0xc9,
0xc3, 0xdc, 0xf4, 0xf6, 0x06, 0xd3, 0x42, 0x3f, 0x24, 0x57, 0x1f, 0xe1, 0xe0, 0x11, 0x7e, 0xb9,
0xe3, 0xbc, 0x22, 0x8f, 0xf6, 0xf4, 0x01, 0x2c, 0x15, 0xed, 0x53, 0xa6, 0x7f, 0x0b, 0x96, 0x25,
0x64, 0x67, 0xe8, 0x5b, 0x3b, 0x22, 0xb7, 0x1a, 0x6d, 0xd4, 0xbf, 0x5b, 0x30, 0x28, 0x13, 0xaa,
0xc8, 0x7f, 0x5d, 0xaf, 0x9d, 0xd8, 0x27, 0x17, 0xa1, 0xc7, 0xb0, 0xe3, 0x9a, 0x74, 0x7f, 0x3f,
0x22, 0x4c, 0x38, 0x62, 0xd2, 0x00, 0x0e, 0x7a, 0x2c, 0x20, 0xe8, 0x2a, 0xcc, 0x5b, 0x32, 0xfa,
0xcd, 0x90, 0x1c, 0x39, 0xe2, 0x34, 0x98, 0x11, 0x8a, 0xcd, 0x59, 0x49, 0x56, 0x48, 0x30, 0xd2,
0x61, 0xd6, 0xb1, 0x5f, 0x9a, 0xe2, 0x38, 0x12, 0x87, 0x49, 0x47, 0x70, 0xeb, 0x39, 0xf6, 0xcb,
0x07, 0x8e, 0x4b, 0xb8, 0x47, 0xf5, 0x27, 0x70, 0x41, 0x1a, 0xff, 0xd0, 0xb7, 0x42, 0xe2, 0x11,
0x9f, 0x61, 0x77, 0x8b, 0x06, 0xc3, 0x46, 0x61, 0x73, 0x0e, 0x3a, 0x91, 0xe3, 0x5b, 0xc4, 0xf4,
0xe5, 0xa1, 0x36, 0x69, 0xcc, 0x88, 0xdf, 0xdb, 0x91, 0x7e, 0x17, 0x56, 0x6a, 0xf8, 0x2a, 0xcf,
0x5e, 0x82, 0xbe, 0x50, 0xcc, 0xa2, 0x3e, 0x23, 0x3e, 0x13, 0xbc, 0xfb, 0x46, 0x8f, 0xc3, 0xb6,
0x24, 0x48, 0x7f, 0x1f, 0x90, 0xe4, 0xf1, 0x88, 0xc6, 0x7e, 0xb3, 0x74, 0x3e, 0x0b, 0x0b, 0x39,
0x12, 0x15, 0x1b, 0xb7, 0x60, 0x51, 0x82, 0x3f, 0xf7, 0xbd, 0xc6, 0xbc, 0x96, 0xe1, 0x6c, 0x81,
0x48, 0x71, 0xdb, 0x4c, 0x84, 0xe4, 0xaf, 0x9d, 0x63, 0x99, 0x2d, 0x25, 0x1a, 0xe4, 0x6f, 0x1e,
0x71, 0x72, 0x49, 0x85, 0x71, 0x78, 0x68, 0x10, 0x6c, 0x53, 0xdf, 0x1d, 0x36, 0x3e, 0xb9, 0x2a,
0x28, 0x15, 0xdf, 0xdf, 0xb7, 0xe0, 0x4c, 0x72, 0xa4, 0x35, 0xdc, 0xcd, 0x13, 0x86, 0x73, 0xbb,
0x36, 0x9c, 0x27, 0x47, 0xe1, 0xbc, 0x0e, 0xf3, 0x11, 0x8d, 0x43, 0x8b, 0x98, 0x36, 0x66, 0xd8,
0xf4, 0xa9, 0x4d, 0x54, 0xb4, 0x9f, 0x96, 0xf0, 0x7b, 0x98, 0xe1, 0x6d, 0x6a, 0x13, 0xfd, 0x7b,
0xc9, 0x66, 0xe7, 0xa2, 0xe4, 0x2a, 0x9c, 0x71, 0x71, 0xc4, 0x4c, 0x1c, 0x04, 0xc4, 0xb7, 0x4d,
0xcc, 0x78, 0xa8, 0xb5, 0x44, 0xa8, 0x9d, 0xe6, 0x0b, 0x77, 0x04, 0xfc, 0x0e, 0xdb, 0x8e, 0xf4,
0xbf, 0xb4, 0x60, 0x8e, 0xd3, 0xf2, 0xd0, 0x6e, 0x64, 0xef, 0x3c, 0xb4, 0xc9, 0x4b, 0xa6, 0x0c,
0xe5, 0x9f, 0xe8, 0x06, 0x2c, 0xa8, 0x1c, 0x72, 0xa8, 0x3f, 0x4a, 0xaf, 0xb6, 0x3c, 0x8d, 0x46,
0x4b, 0x69, 0x86, 0x5d, 0x84, 0x5e, 0xc4, 0x68, 0x90, 0x64, 0xeb, 0xa4, 0xcc, 0x56, 0x0e, 0x52,
0xd9, 0x9a, 0xf7, 0xe9, 0x54, 0x85, 0x4f, 0xfb, 0x4e, 0x64, 0x12, 0xcb, 0x94, 0x5a, 0x89, 0x7c,
0xef, 0x18, 0xe0, 0x44, 0xf7, 0x2d, 0xe9, 0x0d, 0xfd, 0xff, 0x61, 0x7e, 0x64, 0x55, 0xf3, 0xdc,
0xf9, 0xaa, 0x95, 0x1c, 0x87, 0xbb, 0xd8, 0x71, 0x77, 0x88, 0x6f, 0x93, 0xf0, 0x35, 0x73, 0x1a,
0xdd, 0x84, 0x45, 0xc7, 0x76, 0x89, 0xc9, 0x1c, 0x8f, 0xd0, 0x98, 0x99, 0x11, 0xb1, 0xa8, 0x6f,
0x47, 0x89, 0x7f, 0xf8, 0xda, 0xae, 0x5c, 0xda, 0x91, 0x2b, 0xfa, 0xaf, 0xd3, 0xb3, 0x35, 0xab,
0xc5, 0xa8, 0xaa, 0xf0, 0x09, 0xe1, 0x0c, 0x0f, 0x08, 0xb6, 0x49, 0xa8, 0xcc, 0xe8, 0x4b, 0xe0,
0x0f, 0x05, 0x8c, 0x7b, 0x58, 0x21, 0xed, 0x51, 0x7b, 0x28, 0x34, 0xea, 0x1b, 0x20, 0x41, 0x77,
0xa9, 0x3d, 0x14, 0x87, 0x5c, 0x64, 0x8a, 0x20, 0xb1, 0x0e, 0x62, 0xff, 0x50, 0x68, 0xd3, 0x31,
0x7a, 0x4e, 0xf4, 0x29, 0x8e, 0xd8, 0x16, 0x07, 0xe9, 0x7f, 0x6c, 0x25, 0x59, 0xc6, 0xd5, 0x30,
0x88, 0x45, 0x9c, 0xa3, 0xff, 0x82, 0x3b, 0x38, 0x85, 0xca, 0x86, 0x5c, 0x75, 0xa9, 0x12, 0x06,
0xc9, 0x35, 0x75, 0x17, 0x89, 0x95, 0x51, 0x92, 0xe7, 0x15, 0x57, 0x49, 0xfe, 0x65, 0x72, 0xc8,
0xde, 0xb7, 0x76, 0x0e, 0x70, 0x68, 0x47, 0x3f, 0x20, 0x3e, 0x09, 0x31, 0x7b, 0x23, 0x97, 0xbe,
0xbe, 0x06, 0xab, 0x75, 0xdc, 0x95, 0xfc, 0xa7, 0xc9, 0xe5, 0x91, 0x60, 0x18, 0x64, 0x2f, 0x76,
0x5c, 0xfb, 0x8d, 0x88, 0xff, 0xa4, 0x68, 0x5c, 0xca, 0x5c, 0xc5, 0xcf, 0x35, 0x38, 0x13, 0x0a,
0x10, 0x33, 0x23, 0x8e, 0x90, 0xd6, 0xfb, 0xb3, 0xc6, 0x9c, 0x5a, 0x10, 0x84, 0xbc, 0xee, 0xff,
0x53, 0x1a, 0x01, 0x09, 0xb7, 0x37, 0x76, 0x2c, 0x9e, 0x87, 0xee, 0x48, 0x7c, 0x5b, 0x88, 0xef,
0x44, 0x4a, 0x2e, 0x8f, 0x4e, 0x8b, 0x06, 0x43, 0x93, 0x58, 0xf2, 0x1e, 0x16, 0x5b, 0xdd, 0x31,
0x7a, 0x1c, 0x78, 0xdf, 0x12, 0xd7, 0xf0, 0x09, 0xce, 0xc8, 0x34, 0x1a, 0xf2, 0x46, 0xa8, 0xdd,
0x78, 0x01, 0xe7, 0xf3, 0xab, 0xcd, 0xaf, 0xa7, 0xd7, 0x32, 0x52, 0x5f, 0x2d, 0x86, 0x41, 0xe1,
0x8e, 0x3b, 0x2a, 0xaa, 0xdd, 0xf8, 0x3e, 0x7f, 0x3d, 0xbd, 0x56, 0x8a, 0x0e, 0xc9, 0x17, 0x05,
0x5f, 0x14, 0xd5, 0x3e, 0x41, 0x71, 0x70, 0xbc, 0xe0, 0x8b, 0xc5, 0xd0, 0x2d, 0x56, 0x10, 0x5f,
0xa7, 0xe7, 0xa2, 0xc2, 0xe0, 0xf7, 0x77, 0xe3, 0xf3, 0x48, 0xc9, 0x15, 0xee, 0x98, 0x35, 0x66,
0x94, 0x58, 0xfe, 0xc0, 0x54, 0xf7, 0x90, 0xac, 0xcf, 0xd5, 0xaf, 0xdc, 0x53, 0xb2, 0xad, 0x9e,
0x92, 0xc9, 0x13, 0xf9, 0x90, 0x0c, 0x45, 0xac, 0x4d, 0xca, 0x27, 0xf2, 0x27, 0x64, 0xa8, 0x6f,
0x17, 0x32, 0x45, 0xaa, 0xa6, 0x72, 0x0e, 0xc1, 0x24, 0x0f, 0x52, 0x75, 0x54, 0x8b, 0x6f, 0xb4,
0x02, 0xe0, 0x44, 0xa6, 0x2d, 0xf6, 0x5c, 0x2a, 0xd5, 0x31, 0xba, 0x8e, 0x0a, 0x02, 0x5b, 0xff,
0x6d, 0x26, 0xf5, 0xee, 0xba, 0x74, 0xef, 0x0d, 0x46, 0x65, 0xd6, 0x8a, 0x76, 0xce, 0x8a, 0xec,
0x5b, 0x79, 0x32, 0xff, 0x56, 0xce, 0x24, 0x51, 0x56, 0x1d, 0xb5, 0x33, 0x1f, 0xc2, 0x79, 0x6e,
0xb0, 0xc4, 0x10, 0x55, 0x72, 0xf3, 0x97, 0xc4, 0x3f, 0x4e, 0xc1, 0x85, 0x6a, 0xe2, 0x26, 0xaf,
0x89, 0x8f, 0x40, 0x4b, 0xab, 0x75, 0x7e, 0xa5, 0x44, 0x0c, 0x7b, 0x41, 0x7a, 0xa9, 0xc8, 0xbb,
0x67, 0x59, 0x95, 0xee, 0xbb, 0xc9, 0x7a, 0x72, 0xb3, 0x94, 0x4a, 0xfd, 0x76, 0xa9, 0xd4, 0xe7,
0x02, 0x6c, 0xcc, 0xea, 0x04, 0xc8, 0xda, 0x65, 0xd9, 0xc6, 0xac, 0x4e, 0x40, 0x4a, 0x2c, 0x04,
0xc8, 0xa8, 0xe9, 0x29, 0x7c, 0x21, 0x60, 0x05, 0x40, 0x95, 0x25, 0xb1, 0x9f, 0x3c, 0x5d, 0xba,
0xb2, 0x28, 0x89, 0xfd, 0xda, 0xea, 0x6a, 0xa6, 0xb6, 0xba, 0xca, 0x6f, 0x7f, 0xa7, 0x74, 0x43,
0x7c, 0x01, 0x70, 0xcf, 0x89, 0x0e, 0xa5, 0x93, 0x79, 0x39, 0x67, 0x3b, 0xa1, 0x7a, 0x2f, 0xf3,
0x4f, 0x0e, 0xc1, 0xae, 0xab, 0x5c, 0xc7, 0x3f, 0x79, 0xf8, 0xc6, 0x11, 0xb1, 0x95, 0x77, 0xc4,
0x37, 0x87, 0xed, 0x87, 0x84, 0x28, 0x07, 0x88, 0x6f, 0xfd, 0x77, 0x2d, 0xe8, 0x3e, 0x22, 0x9e,
0xe2, 0xbc, 0x0a, 0xf0, 0x8c, 0x86, 0x34, 0x66, 0x8e, 0x4f, 0x64, 0xf5, 0x39, 0x65, 0x64, 0x20,
0xdf, 0x5c, 0x8e, 0x48, 0x4d, 0xe2, 0xee, 0x2b, 0x67, 0x8a, 0x6f, 0x0e, 0x3b, 0x20, 0x38, 0x50,
0xfe, 0x13, 0xdf, 0x68, 0x11, 0xa6, 0x22, 0x86, 0xad, 0x43, 0xe1, 0xac, 0x49, 0x43, 0xfe, 0xd0,
0x7d, 0xe8, 0xef, 0x3a, 0x24, 0x24, 0x2a, 0xe0, 0x78, 0x59, 0xb8, 0x87, 0xad, 0x43, 0x5e, 0x28,
0xb3, 0x61, 0x40, 0x94, 0x2b, 0x7a, 0x0a, 0xb6, 0x3b, 0x0c, 0x72, 0x28, 0x3e, 0xf6, 0x88, 0xca,
0xa9, 0x04, 0x65, 0x1b, 0x7b, 0xb9, 0x2e, 0x93, 0xca, 0xa9, 0x24, 0x73, 0xbe, 0x6e, 0xc1, 0x9a,
0xaa, 0x46, 0x1c, 0x12, 0xf2, 0xbb, 0xe7, 0x1e, 0x66, 0xbb, 0xd4, 0x20, 0x1e, 0x7d, 0x43, 0x09,
0x7d, 0x1b, 0x06, 0x36, 0x89, 0x98, 0xe3, 0x8b, 0xf7, 0x84, 0x99, 0x53, 0x55, 0xbe, 0x37, 0x96,
0x32, 0xeb, 0x77, 0x47, 0x5a, 0xeb, 0x97, 0xe1, 0xd2, 0x31, 0xaa, 0xa9, 0xe4, 0xfe, 0x57, 0x1f,
0xfa, 0x9f, 0xc5, 0x24, 0x1c, 0x66, 0x5a, 0x2d, 0x11, 0x51, 0xc2, 0x93, 0x5e, 0x61, 0x06, 0xc2,
0xa3, 0x7e, 0x3f, 0xa4, 0x9e, 0x99, 0xb6, 0x13, 0x4f, 0x09, 0x94, 0x1e, 0x07, 0x3e, 0x90, 0x2d,
0x45, 0xf4, 0x31, 0x4c, 0xef, 0x3b, 0x2e, 0x23, 0xb2, 0x81, 0xd7, 0xdb, 0x7c, 0xbb, 0xdc, 0x3a,
0xcc, 0xca, 0xdc, 0x78, 0x20, 0x90, 0x0d, 0x45, 0x84, 0xf6, 0x60, 0xc1, 0xf1, 0x03, 0x51, 0x3e,
0x86, 0x0e, 0x76, 0x9d, 0x57, 0xa3, 0x66, 0x41, 0x6f, 0xf3, 0xfd, 0x31, 0xbc, 0x1e, 0x72, 0xca,
0x9d, 0x2c, 0xa1, 0x81, 0x9c, 0x12, 0x0c, 0x11, 0x58, 0xa4, 0x31, 0x2b, 0x0b, 0x99, 0x12, 0x42,
0x36, 0xc7, 0x08, 0x79, 0x2c, 0x48, 0xf3, 0x52, 0x16, 0x68, 0x19, 0xa8, 0x6d, 0xc3, 0xb4, 0x34,
0x8e, 0xc7, 0xeb, 0xbe, 0x43, 0xdc, 0xa4, 0x05, 0x2a, 0x7f, 0xf0, 0xc8, 0xa2, 0x01, 0x09, 0xb1,
0x6f, 0xab, 0xad, 0x4f, 0x7e, 0x72, 0xfc, 0x23, 0xec, 0xc6, 0xc9, 0x26, 0xcb, 0x1f, 0xda, 0x5f,
0xa7, 0x00, 0x95, 0x2d, 0x4c, 0x3a, 0x20, 0x21, 0x89, 0x78, 0x54, 0x66, 0x43, 0x7d, 0x2e, 0x03,
0x17, 0xe1, 0xfe, 0x13, 0xe8, 0x5a, 0xd1, 0x91, 0x29, 0x5c, 0x22, 0x64, 0xf6, 0x36, 0x3f, 0x3c,
0xb1, 0x4b, 0x37, 0xb6, 0x76, 0x9e, 0x08, 0xa8, 0xd1, 0xb1, 0xa2, 0x23, 0xf1, 0x85, 0x7e, 0x06,
0xf0, 0x8b, 0x88, 0xfa, 0x8a, 0xb3, 0xdc, 0xf8, 0x8f, 0x4e, 0xce, 0xf9, 0x47, 0x3b, 0x8f, 0xb7,
0x25, 0xeb, 0x2e, 0x67, 0x27, 0x79, 0x5b, 0x30, 0x1b, 0xe0, 0xf0, 0x79, 0x4c, 0x98, 0x62, 0x2f,
0x63, 0xe1, 0xbb, 0x27, 0x67, 0xff, 0x63, 0xc9, 0x46, 0x4a, 0xe8, 0x07, 0x99, 0x5f, 0xda, 0x9f,
0x4f, 0x41, 0x27, 0xb1, 0x8b, 0x57, 0xa0, 0x22, 0xc2, 0xe5, 0x3b, 0xcc, 0x74, 0xfc, 0x7d, 0xaa,
0x3c, 0x7a, 0x9a, 0xc3, 0xe5, 0x53, 0xec, 0xa1, 0xbf, 0x4f, 0xb9, 0xef, 0x43, 0x62, 0xd1, 0xd0,
0xe6, 0xf7, 0xbd, 0xe3, 0x39, 0x3c, 0xec, 0xe5, 0x5e, 0xce, 0x49, 0xf8, 0xbd, 0x04, 0x8c, 0xae,
0xc0, 0x9c, 0xd8, 0xf6, 0x0c, 0x66, 0x3b, 0xe1, 0x49, 0xdc, 0x0c, 0xe2, 0x55, 0x98, 0x7f, 0x1e,
0x53, 0x46, 0x4c, 0xeb, 0x00, 0x87, 0xd8, 0x62, 0x34, 0x7d, 0x11, 0xcd, 0x09, 0xf8, 0x56, 0x0a,
0x46, 0xff, 0x07, 0x4b, 0x12, 0x95, 0x44, 0x16, 0x0e, 0x52, 0x0a, 0x12, 0xaa, 0x82, 0x79, 0x51,
0xac, 0xde, 0x17, 0x8b, 0x5b, 0xc9, 0x1a, 0xd2, 0xa0, 0x63, 0x51, 0xcf, 0x23, 0x3e, 0x8b, 0xc4,
0xa9, 0xda, 0x35, 0xd2, 0xdf, 0xe8, 0x0e, 0xac, 0x60, 0xd7, 0xa5, 0x2f, 0x4c, 0x41, 0x69, 0x9b,
0x25, 0xeb, 0x66, 0x44, 0x3d, 0xa3, 0x09, 0xa4, 0xcf, 0x04, 0x8e, 0x91, 0x37, 0x54, 0xbb, 0x08,
0xdd, 0x74, 0x1f, 0xf9, 0xe9, 0x9d, 0x09, 0x48, 0xf1, 0xad, 0x9d, 0x86, 0x7e, 0x76, 0x27, 0xb4,
0x7f, 0xb6, 0x61, 0xa1, 0x22, 0xa9, 0xd0, 0x53, 0x00, 0x1e, 0xad, 0x32, 0xb5, 0x54, 0xb8, 0x7e,
0xe7, 0xe4, 0xc9, 0xc9, 0xe3, 0x55, 0x82, 0x0d, 0x1e, 0xfd, 0xf2, 0x13, 0xfd, 0x1c, 0x7a, 0x22,
0x62, 0x15, 0x77, 0x19, 0xb2, 0x1f, 0x7f, 0x03, 0xee, 0xdc, 0x56, 0xc5, 0x5e, 0xe4, 0x80, 0xfc,
0xd6, 0xfe, 0xde, 0x82, 0x6e, 0x2a, 0x98, 0xdf, 0x33, 0x72, 0xa3, 0xc4, 0x5e, 0x47, 0xc9, 0x55,
0x24, 0x60, 0x0f, 0x04, 0xe8, 0x7f, 0x32, 0x94, 0xb4, 0x0f, 0x00, 0x46, 0xf6, 0x57, 0x9a, 0xd0,
0xaa, 0x34, 0x41, 0xbf, 0x0a, 0xb3, 0xdc, 0xb3, 0x0e, 0xb1, 0x77, 0x58, 0xe8, 0x04, 0xe2, 0x9a,
0x95, 0x38, 0x91, 0x2a, 0xa6, 0x93, 0x9f, 0x9b, 0x7f, 0x18, 0x40, 0x3f, 0xdb, 0x04, 0x40, 0x5f,
0x42, 0x2f, 0x33, 0xb4, 0x42, 0x6f, 0x95, 0x37, 0xad, 0x3c, 0x04, 0xd3, 0xde, 0x1e, 0x83, 0xa5,
0xae, 0xc4, 0x09, 0xe4, 0xc3, 0x99, 0xd2, 0xe4, 0x07, 0x5d, 0x2b, 0x53, 0xd7, 0xcd, 0x95, 0xb4,
0xeb, 0x8d, 0x70, 0x53, 0x79, 0x0c, 0x16, 0x2a, 0x46, 0x39, 0xe8, 0xdd, 0x31, 0x5c, 0x72, 0xe3,
0x24, 0xed, 0xbd, 0x86, 0xd8, 0xa9, 0xd4, 0xe7, 0x80, 0xca, 0x73, 0x1e, 0x74, 0x7d, 0x2c, 0x9b,
0xd1, 0x1c, 0x49, 0x7b, 0xb7, 0x19, 0x72, 0xad, 0xa1, 0x72, 0x02, 0x34, 0xd6, 0xd0, 0xdc, 0x8c,
0x69, 0xac, 0xa1, 0x85, 0xb1, 0xd2, 0x04, 0x3a, 0x84, 0xf9, 0xe2, 0x74, 0x08, 0x5d, 0xad, 0x9b,
0x66, 0x96, 0x86, 0x4f, 0xda, 0xb5, 0x26, 0xa8, 0xa9, 0x30, 0x02, 0xa7, 0xf3, 0xd3, 0x18, 0x74,
0xa5, 0x4c, 0x5f, 0x39, 0x8f, 0xd2, 0xd6, 0xc7, 0x23, 0x66, 0x6d, 0x2a, 0x4e, 0x68, 0xaa, 0x6c,
0xaa, 0x19, 0xff, 0x54, 0xd9, 0x54, 0x37, 0xf0, 0xd1, 0x27, 0xd0, 0x2f, 0x93, 0xb6, 0x7f, 0x61,
0x72, 0x81, 0x36, 0xea, 0xd8, 0x54, 0x8f, 0x4e, 0xb4, 0x1b, 0x8d, 0xf1, 0x13, 0xd9, 0x37, 0x5b,
0x3c, 0xd7, 0x33, 0x03, 0x8c, 0xaa, 0x5c, 0x2f, 0x8f, 0x44, 0xaa, 0x72, 0xbd, 0x6a, 0x0a, 0x32,
0x81, 0xf6, 0x60, 0x36, 0x37, 0xd2, 0x40, 0xef, 0xd4, 0x51, 0xe6, 0x7b, 0x21, 0xda, 0x95, 0xb1,
0x78, 0xa9, 0x0c, 0x33, 0x39, 0xbd, 0xd4, 0x71, 0x55, 0xab, 0x5c, 0xfe, 0xbc, 0x7a, 0x67, 0x1c,
0x5a, 0x2e, 0x95, 0x4b, 0x83, 0x8f, 0xca, 0x54, 0xae, 0x1b, 0xac, 0x54, 0xa6, 0x72, 0xfd, 0x2c,
0x65, 0x02, 0xfd, 0x14, 0x60, 0x34, 0x9c, 0x40, 0x97, 0xeb, 0xa8, 0xb3, 0xbb, 0xff, 0xd6, 0xf1,
0x48, 0x29, 0xeb, 0x17, 0xb0, 0x58, 0xd5, 0x33, 0x40, 0x15, 0x89, 0x7f, 0x4c, 0x63, 0x42, 0xdb,
0x68, 0x8a, 0x9e, 0x0a, 0xfe, 0x1c, 0x3a, 0xc9, 0x60, 0x01, 0x5d, 0x2a, 0x53, 0x17, 0x46, 0x29,
0x9a, 0x7e, 0x1c, 0x4a, 0x26, 0x80, 0xbd, 0x24, 0x57, 0x47, 0x1d, 0xff, 0xfa, 0x5c, 0x2d, 0xcd,
0x26, 0xea, 0x73, 0xb5, 0x3c, 0x40, 0x10, 0xe2, 0xd2, 0x60, 0xc8, 0x36, 0xc8, 0xeb, 0x83, 0xa1,
0xa2, 0xff, 0x5f, 0x1f, 0x0c, 0x95, 0x3d, 0xf7, 0x09, 0xf4, 0x2b, 0x58, 0xaa, 0xee, 0x8b, 0xa3,
0xda, 0x8c, 0xaf, 0xe9, 0xcf, 0x6b, 0x37, 0x9b, 0x13, 0xa4, 0xe2, 0x5f, 0x25, 0xe7, 0x53, 0xa1,
0x2f, 0x5e, 0x7f, 0x3e, 0x55, 0x77, 0xe7, 0xb5, 0x1b, 0x8d, 0xf1, 0xcb, 0xa9, 0x97, 0x6d, 0x40,
0xd7, 0x7b, 0xbb, 0xa2, 0xd7, 0x5e, 0xef, 0xed, 0xca, 0x9e, 0xb6, 0xc8, 0x8f, 0xaa, 0xe6, 0x72,
0x55, 0x7e, 0x1c, 0xd3, 0xfd, 0xd6, 0x36, 0x9a, 0xa2, 0xe7, 0xae, 0xef, 0x72, 0xf7, 0x18, 0x8d,
0xd5, 0x3f, 0x77, 0x32, 0xbf, 0xd7, 0x10, 0xbb, 0x7e, 0x77, 0x93, 0x93, 0x7a, 0xac, 0x01, 0x85,
0x13, 0xfb, 0x46, 0x63, 0xfc, 0x54, 0x76, 0x90, 0x8c, 0x8c, 0x33, 0x9d, 0x5f, 0x74, 0x6d, 0x0c,
0x9f, 0x4c, 0xe7, 0x5a, 0xbb, 0xde, 0x08, 0xb7, 0x2a, 0x7b, 0xb3, 0xbd, 0xd8, 0xe3, 0xe2, 0xa9,
0xd4, 0x40, 0x3e, 0x2e, 0x9e, 0x2a, 0xda, 0xbb, 0x13, 0xe8, 0x37, 0xa3, 0x59, 0x60, 0xb9, 0x53,
0x84, 0x36, 0x6b, 0xcf, 0x82, 0xda, 0x8e, 0x97, 0x76, 0xeb, 0x44, 0x34, 0xa9, 0x22, 0x9f, 0xc2,
0x94, 0x78, 0x6b, 0xa1, 0xd5, 0xe3, 0x1f, 0x61, 0xda, 0xc5, 0xea, 0xf5, 0xf4, 0x29, 0xc1, 0x3d,
0xb9, 0x37, 0x2d, 0xfe, 0x22, 0x77, 0xeb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x81, 0x26, 0xf7,
0x49, 0x39, 0x27, 0x00, 0x00,
0xf5, 0x5a, 0xaf, 0x64, 0xed, 0xbe, 0x5d, 0x59, 0x72, 0x4b, 0x91, 0x37, 0x23, 0x4b, 0x56, 0x26,
0x5f, 0x92, 0x13, 0xcb, 0x8e, 0xf2, 0xfb, 0x91, 0x90, 0x10, 0xc0, 0x96, 0x6c, 0x30, 0x89, 0xe5,
0x64, 0xe4, 0x98, 0x40, 0x52, 0x4c, 0xb5, 0x66, 0x5a, 0xd6, 0xa0, 0x99, 0xe9, 0xf1, 0x4c, 0xaf,
0xe2, 0x75, 0xc1, 0x29, 0x1c, 0xb8, 0xf0, 0x07, 0x70, 0xe6, 0xce, 0x89, 0x2a, 0xaa, 0xb8, 0x72,
0xc9, 0x1f, 0x00, 0x57, 0x2e, 0x9c, 0x39, 0x70, 0xa3, 0x8a, 0x0b, 0xd5, 0x5f, 0xb3, 0xf3, 0xa9,
0x1d, 0xc5, 0xae, 0xa2, 0xb8, 0xf5, 0xbc, 0x7e, 0x1f, 0xfd, 0x5e, 0xbf, 0xf7, 0xfa, 0x75, 0xbf,
0x81, 0xc5, 0x13, 0xea, 0x0f, 0x03, 0x62, 0x27, 0x24, 0x3e, 0x21, 0xf1, 0x56, 0x14, 0x53, 0x46,
0xd1, 0x42, 0x0e, 0x68, 0x47, 0x07, 0xe6, 0x75, 0x40, 0xb7, 0x30, 0x73, 0x8e, 0x76, 0x89, 0x4f,
0x18, 0xb1, 0xc8, 0xe3, 0x21, 0x49, 0x18, 0x7a, 0x11, 0x3a, 0x87, 0x9e, 0x4f, 0x6c, 0xcf, 0x4d,
0x06, 0xad, 0xf5, 0xf6, 0x46, 0xd7, 0x9a, 0xe5, 0xdf, 0x77, 0xdd, 0xc4, 0xbc, 0x0f, 0x8b, 0x39,
0x82, 0x24, 0xa2, 0x61, 0x42, 0xd0, 0xbb, 0x30, 0x1b, 0x93, 0x64, 0xe8, 0x33, 0x49, 0xd0, 0xdb,
0x5e, 0xdb, 0x2a, 0xca, 0xda, 0x4a, 0x49, 0x86, 0x3e, 0xb3, 0x34, 0xba, 0xf9, 0x55, 0x0b, 0xfa,
0xd9, 0x19, 0x74, 0x09, 0x66, 0x95, 0xf0, 0x41, 0x6b, 0xbd, 0xb5, 0xd1, 0xb5, 0xce, 0x4b, 0xd9,
0x68, 0x19, 0xce, 0x27, 0x0c, 0xb3, 0x61, 0x32, 0x38, 0xb7, 0xde, 0xda, 0x98, 0xb1, 0xd4, 0x17,
0x5a, 0x82, 0x19, 0x12, 0xc7, 0x34, 0x1e, 0xb4, 0x05, 0xba, 0xfc, 0x40, 0x08, 0xa6, 0x13, 0xef,
0x29, 0x19, 0x4c, 0xaf, 0xb7, 0x36, 0xe6, 0x2c, 0x31, 0x46, 0x03, 0x98, 0x3d, 0x21, 0x71, 0xe2,
0xd1, 0x70, 0x30, 0x23, 0xc0, 0xfa, 0xd3, 0x9c, 0x85, 0x99, 0xdb, 0x41, 0xc4, 0x46, 0xe6, 0x3b,
0x30, 0x78, 0x88, 0x9d, 0xe1, 0x30, 0x78, 0x28, 0x96, 0xbf, 0x73, 0x44, 0x9c, 0x63, 0x6d, 0x96,
0x15, 0xe8, 0x2a, 0xa5, 0xd4, 0xda, 0xe6, 0xac, 0x8e, 0x04, 0xdc, 0x75, 0xcd, 0xef, 0xc3, 0x8b,
0x15, 0x84, 0xca, 0x3c, 0x2f, 0xc3, 0xdc, 0x23, 0x1c, 0x1f, 0xe0, 0x47, 0xc4, 0x8e, 0x31, 0xf3,
0xa8, 0xa0, 0x6e, 0x59, 0x7d, 0x05, 0xb4, 0x38, 0xcc, 0xfc, 0x1c, 0x8c, 0x1c, 0x07, 0x1a, 0x44,
0xd8, 0x61, 0x4d, 0x84, 0xa3, 0x75, 0xe8, 0x45, 0x31, 0xc1, 0xbe, 0x4f, 0x1d, 0xcc, 0x88, 0xb0,
0x4f, 0xdb, 0xca, 0x82, 0xcc, 0x55, 0x58, 0xa9, 0x64, 0x2e, 0x17, 0x68, 0xbe, 0x5b, 0x58, 0x3d,
0x0d, 0x02, 0xaf, 0x91, 0x68, 0xf3, 0x72, 0x69, 0xd5, 0x82, 0x52, 0xf1, 0xfd, 0x76, 0x61, 0xd6,
0x27, 0x38, 0x1c, 0x46, 0x8d, 0x18, 0x17, 0x57, 0xac, 0x49, 0x53, 0xce, 0x97, 0xa4, 0xdb, 0xec,
0x50, 0xdf, 0x27, 0x0e, 0xf3, 0x68, 0xa8, 0xd9, 0xae, 0x01, 0x38, 0x29, 0x50, 0x39, 0x51, 0x06,
0x62, 0x1a, 0x30, 0x28, 0x93, 0x2a, 0xb6, 0x7f, 0x6b, 0xc1, 0x0b, 0x37, 0x95, 0xd1, 0xa4, 0xe0,
0x46, 0x1b, 0x90, 0x17, 0x79, 0xae, 0x28, 0xb2, 0xb8, 0x41, 0xed, 0xd2, 0x06, 0x71, 0x8c, 0x98,
0x44, 0xbe, 0xe7, 0x60, 0xc1, 0x62, 0x5a, 0xb0, 0xc8, 0x82, 0xd0, 0x02, 0xb4, 0x19, 0xf3, 0x85,
0xe7, 0x76, 0x2d, 0x3e, 0x44, 0xdb, 0xb0, 0x1c, 0x90, 0x80, 0xc6, 0x23, 0x3b, 0xc0, 0x91, 0x1d,
0xe0, 0x27, 0x36, 0x77, 0x73, 0x3b, 0x38, 0x18, 0x9c, 0x17, 0xeb, 0x43, 0x72, 0xf6, 0x1e, 0x8e,
0xee, 0xe1, 0x27, 0xfb, 0xde, 0x53, 0x72, 0xef, 0xc0, 0x1c, 0xc0, 0x72, 0x51, 0x3f, 0xa5, 0xfa,
0xb7, 0xe0, 0x92, 0x84, 0xec, 0x8f, 0x42, 0x67, 0x5f, 0xc4, 0x56, 0xa3, 0x8d, 0xfa, 0x77, 0x0b,
0x06, 0x65, 0x42, 0xe5, 0xf9, 0xcf, 0x6a, 0xb5, 0x33, 0xdb, 0xe4, 0x0a, 0xf4, 0x18, 0xf6, 0x7c,
0x9b, 0x1e, 0x1e, 0x26, 0x84, 0x09, 0x43, 0x4c, 0x5b, 0xc0, 0x41, 0xf7, 0x05, 0x04, 0x6d, 0xc2,
0x82, 0x23, 0xbd, 0xdf, 0x8e, 0xc9, 0x89, 0x27, 0xb2, 0xc1, 0xac, 0x58, 0xd8, 0xbc, 0xa3, 0xa3,
0x42, 0x82, 0x91, 0x09, 0x73, 0x9e, 0xfb, 0xc4, 0x16, 0xe9, 0x48, 0x24, 0x93, 0x8e, 0xe0, 0xd6,
0xf3, 0xdc, 0x27, 0x77, 0x3c, 0x9f, 0x70, 0x8b, 0x9a, 0x0f, 0xe1, 0xb2, 0x54, 0xfe, 0x6e, 0xe8,
0xc4, 0x24, 0x20, 0x21, 0xc3, 0xfe, 0x0e, 0x8d, 0x46, 0x8d, 0xdc, 0xe6, 0x45, 0xe8, 0x24, 0x5e,
0xe8, 0x10, 0x3b, 0x94, 0x49, 0x6d, 0xda, 0x9a, 0x15, 0xdf, 0x7b, 0x89, 0x79, 0x0b, 0x56, 0x6b,
0xf8, 0x2a, 0xcb, 0xbe, 0x04, 0x7d, 0xb1, 0x30, 0x87, 0x86, 0x8c, 0x84, 0x4c, 0xf0, 0xee, 0x5b,
0x3d, 0x0e, 0xdb, 0x91, 0x20, 0xf3, 0x2d, 0x40, 0x92, 0xc7, 0x3d, 0x3a, 0x0c, 0x9b, 0x85, 0xf3,
0x0b, 0xb0, 0x98, 0x23, 0x51, 0xbe, 0xf1, 0x36, 0x2c, 0x49, 0xf0, 0xa7, 0x61, 0xd0, 0x98, 0xd7,
0x25, 0x78, 0xa1, 0x40, 0xa4, 0xb8, 0x6d, 0x6b, 0x21, 0xf9, 0x63, 0xe7, 0x54, 0x66, 0xcb, 0x7a,
0x05, 0xf9, 0x93, 0x47, 0x64, 0x2e, 0xb9, 0x60, 0x1c, 0x1f, 0x5b, 0x04, 0xbb, 0x34, 0xf4, 0x47,
0x8d, 0x33, 0x57, 0x05, 0xa5, 0xe2, 0xfb, 0xfb, 0x16, 0x5c, 0xd4, 0x29, 0xad, 0xe1, 0x6e, 0x9e,
0xd1, 0x9d, 0xdb, 0xb5, 0xee, 0x3c, 0x3d, 0x76, 0xe7, 0x0d, 0x58, 0x48, 0xe8, 0x30, 0x76, 0x88,
0xed, 0x62, 0x86, 0xed, 0x90, 0xba, 0x44, 0x79, 0xfb, 0x05, 0x09, 0xdf, 0xc5, 0x0c, 0xef, 0x51,
0x97, 0x98, 0xdf, 0xd3, 0x9b, 0x9d, 0xf3, 0x92, 0x4d, 0xb8, 0xe8, 0xe3, 0x84, 0xd9, 0x38, 0x8a,
0x48, 0xe8, 0xda, 0x98, 0x71, 0x57, 0x6b, 0x09, 0x57, 0xbb, 0xc0, 0x27, 0x6e, 0x0a, 0xf8, 0x4d,
0xb6, 0x97, 0x98, 0x7f, 0x69, 0xc1, 0x3c, 0xa7, 0xe5, 0xae, 0xdd, 0x48, 0xdf, 0x05, 0x68, 0x93,
0x27, 0x4c, 0x29, 0xca, 0x87, 0xe8, 0x3a, 0x2c, 0xaa, 0x18, 0xf2, 0x68, 0x38, 0x0e, 0xaf, 0xb6,
0xcc, 0x46, 0xe3, 0xa9, 0x34, 0xc2, 0xae, 0x40, 0x2f, 0x61, 0x34, 0xd2, 0xd1, 0x3a, 0x2d, 0xa3,
0x95, 0x83, 0x54, 0xb4, 0xe6, 0x6d, 0x3a, 0x53, 0x61, 0xd3, 0xbe, 0x97, 0xd8, 0xc4, 0xb1, 0xe5,
0xaa, 0x44, 0xbc, 0x77, 0x2c, 0xf0, 0x92, 0xdb, 0x8e, 0xb4, 0x86, 0xf9, 0xff, 0xb0, 0x30, 0xd6,
0xaa, 0x79, 0xec, 0x7c, 0xd5, 0xd2, 0xe9, 0xf0, 0x01, 0xf6, 0xfc, 0x7d, 0x12, 0xba, 0x24, 0x7e,
0xc6, 0x98, 0x46, 0x37, 0x60, 0xc9, 0x73, 0x7d, 0x62, 0x33, 0x2f, 0x20, 0x74, 0xc8, 0xec, 0x84,
0x38, 0x34, 0x74, 0x13, 0x6d, 0x1f, 0x3e, 0xf7, 0x40, 0x4e, 0xed, 0xcb, 0x19, 0xf3, 0x57, 0x69,
0x6e, 0xcd, 0xae, 0x62, 0x5c, 0x55, 0x84, 0x84, 0x70, 0x86, 0x47, 0x04, 0xbb, 0x24, 0x56, 0x6a,
0xf4, 0x25, 0xf0, 0x87, 0x02, 0xc6, 0x2d, 0xac, 0x90, 0x0e, 0xa8, 0x3b, 0x12, 0x2b, 0xea, 0x5b,
0x20, 0x41, 0xb7, 0xa8, 0x3b, 0x12, 0x49, 0x2e, 0xb1, 0x85, 0x93, 0x38, 0x47, 0xc3, 0xf0, 0x58,
0xac, 0xa6, 0x63, 0xf5, 0xbc, 0xe4, 0x23, 0x9c, 0xb0, 0x1d, 0x0e, 0x32, 0xff, 0xd8, 0xd2, 0x51,
0xc6, 0x97, 0x61, 0x11, 0x87, 0x78, 0x27, 0xff, 0x05, 0x73, 0x70, 0x0a, 0x15, 0x0d, 0xb9, 0xea,
0x52, 0x05, 0x0c, 0x92, 0x73, 0xea, 0x2c, 0x12, 0x33, 0xe3, 0x20, 0xcf, 0x2f, 0x5c, 0x05, 0xf9,
0x17, 0x3a, 0xc9, 0xde, 0x76, 0xf6, 0x8f, 0x70, 0xec, 0x26, 0x3f, 0x20, 0x21, 0x89, 0x31, 0x7b,
0x2e, 0x87, 0xbe, 0xb9, 0x0e, 0x6b, 0x75, 0xdc, 0x95, 0xfc, 0xcf, 0xf5, 0xe1, 0xa1, 0x31, 0x2c,
0x72, 0x30, 0xf4, 0x7c, 0xf7, 0xb9, 0x88, 0xff, 0xb0, 0xa8, 0x5c, 0xca, 0x5c, 0xf9, 0xcf, 0x55,
0xb8, 0x18, 0x0b, 0x10, 0xb3, 0x13, 0x8e, 0x90, 0xd6, 0xfb, 0x73, 0xd6, 0xbc, 0x9a, 0x10, 0x84,
0xbc, 0xee, 0xff, 0x73, 0xea, 0x01, 0x9a, 0xdb, 0x73, 0x4b, 0x8b, 0x2b, 0xd0, 0x1d, 0x8b, 0x6f,
0x0b, 0xf1, 0x9d, 0x44, 0xc9, 0xe5, 0xde, 0xe9, 0xd0, 0x68, 0x64, 0x13, 0x47, 0x9e, 0xc3, 0x62,
0xab, 0x3b, 0x56, 0x8f, 0x03, 0x6f, 0x3b, 0xe2, 0x18, 0x3e, 0x43, 0x8e, 0x4c, 0xbd, 0x21, 0xaf,
0x84, 0xda, 0x8d, 0x2f, 0x61, 0x25, 0x3f, 0xdb, 0xfc, 0x78, 0x7a, 0x26, 0x25, 0xcd, 0xb5, 0xa2,
0x1b, 0x14, 0xce, 0xb8, 0x93, 0xe2, 0xb2, 0x1b, 0x9f, 0xe7, 0xcf, 0xb6, 0xae, 0xd5, 0xa2, 0x41,
0xf2, 0x45, 0xc1, 0x67, 0xc5, 0x65, 0x9f, 0xa1, 0x38, 0x38, 0x5d, 0xf0, 0x95, 0xa2, 0xeb, 0x16,
0x2b, 0x88, 0xdf, 0xa6, 0x79, 0x51, 0x61, 0xf0, 0xf3, 0xbb, 0x71, 0x3e, 0x52, 0x72, 0x85, 0x39,
0xe6, 0xac, 0x59, 0x25, 0x96, 0x5f, 0x30, 0xd5, 0x39, 0x24, 0xeb, 0x73, 0xf5, 0x95, 0xbb, 0x4a,
0xb6, 0xd5, 0x55, 0x52, 0x5f, 0x91, 0x8f, 0xc9, 0x48, 0xf8, 0xda, 0xb4, 0xbc, 0x22, 0x7f, 0x48,
0x46, 0xe6, 0x5e, 0x21, 0x52, 0xe4, 0xd2, 0x54, 0xcc, 0x21, 0x98, 0xe6, 0x4e, 0xaa, 0x52, 0xb5,
0x18, 0xa3, 0x55, 0x00, 0x2f, 0xb1, 0x5d, 0xb1, 0xe7, 0x72, 0x51, 0x1d, 0xab, 0xeb, 0x29, 0x27,
0x70, 0xcd, 0xdf, 0x64, 0x42, 0xef, 0x96, 0x4f, 0x0f, 0x9e, 0xa3, 0x57, 0x66, 0xb5, 0x68, 0xe7,
0xb4, 0xc8, 0xde, 0x95, 0xa7, 0xf3, 0x77, 0xe5, 0x4c, 0x10, 0x65, 0x97, 0xa3, 0x76, 0xe6, 0x3d,
0x58, 0xe1, 0x0a, 0x4b, 0x0c, 0x51, 0x25, 0x37, 0xbf, 0x49, 0xfc, 0xe3, 0x1c, 0x5c, 0xae, 0x26,
0x6e, 0x72, 0x9b, 0x78, 0x1f, 0x8c, 0xb4, 0x5a, 0xe7, 0x47, 0x4a, 0xc2, 0x70, 0x10, 0xa5, 0x87,
0x8a, 0x3c, 0x7b, 0x2e, 0xa9, 0xd2, 0xfd, 0x81, 0x9e, 0xd7, 0x27, 0x4b, 0xa9, 0xd4, 0x6f, 0x97,
0x4a, 0x7d, 0x2e, 0xc0, 0xc5, 0xac, 0x4e, 0x80, 0xac, 0x5d, 0x2e, 0xb9, 0x98, 0xd5, 0x09, 0x48,
0x89, 0x85, 0x00, 0xe9, 0x35, 0x3d, 0x85, 0x2f, 0x04, 0xac, 0x02, 0xa8, 0xb2, 0x64, 0x18, 0xea,
0xab, 0x4b, 0x57, 0x16, 0x25, 0xc3, 0xb0, 0xb6, 0xba, 0x9a, 0xad, 0xad, 0xae, 0xf2, 0xdb, 0xdf,
0x29, 0x9d, 0x10, 0x9f, 0x01, 0xec, 0x7a, 0xc9, 0xb1, 0x34, 0x32, 0x2f, 0xe7, 0x5c, 0x2f, 0x56,
0xf7, 0x65, 0x3e, 0xe4, 0x10, 0xec, 0xfb, 0xca, 0x74, 0x7c, 0xc8, 0xdd, 0x77, 0x98, 0x10, 0x57,
0x59, 0x47, 0x8c, 0x39, 0xec, 0x30, 0x26, 0x44, 0x19, 0x40, 0x8c, 0xcd, 0xdf, 0xb5, 0xa0, 0x7b,
0x8f, 0x04, 0x8a, 0xf3, 0x1a, 0xc0, 0x23, 0x1a, 0xd3, 0x21, 0xf3, 0x42, 0x22, 0xab, 0xcf, 0x19,
0x2b, 0x03, 0xf9, 0xe6, 0x72, 0x44, 0x68, 0x12, 0xff, 0x50, 0x19, 0x53, 0x8c, 0x39, 0xec, 0x88,
0xe0, 0x48, 0xd9, 0x4f, 0x8c, 0xd1, 0x12, 0xcc, 0x24, 0x0c, 0x3b, 0xc7, 0xc2, 0x58, 0xd3, 0x96,
0xfc, 0x30, 0xff, 0xd4, 0x02, 0xb0, 0x48, 0x40, 0x99, 0xf0, 0x35, 0x5e, 0x15, 0x1e, 0x60, 0xe7,
0x98, 0xd7, 0xc9, 0x6c, 0x14, 0x11, 0x65, 0x89, 0x9e, 0x82, 0x3d, 0x18, 0x45, 0x62, 0x87, 0x34,
0x8a, 0xca, 0x1f, 0x5d, 0xab, 0xab, 0x20, 0xb2, 0x22, 0xd6, 0xa1, 0xd4, 0xb5, 0xf8, 0x30, 0x93,
0x53, 0xe4, 0xb2, 0x75, 0x4e, 0x59, 0x81, 0x6e, 0xd1, 0x15, 0x44, 0x28, 0x0a, 0x3f, 0x78, 0x19,
0xe6, 0x02, 0xea, 0x7a, 0x87, 0x1e, 0x71, 0x85, 0xa3, 0x29, 0x55, 0xfa, 0x1a, 0xc8, 0x9d, 0xcb,
0xdc, 0x85, 0x0b, 0xaa, 0xb2, 0xf1, 0x48, 0x7c, 0x37, 0x3c, 0xa4, 0x68, 0x1b, 0x66, 0x38, 0x0b,
0xfd, 0x04, 0x77, 0xb9, 0xfc, 0x04, 0x37, 0x56, 0xd6, 0x92, 0xa8, 0xe6, 0xd7, 0x2d, 0x58, 0x1f,
0xb3, 0xe1, 0xc7, 0xe1, 0x2e, 0x66, 0x0f, 0xa8, 0x44, 0x7c, 0x2e, 0x39, 0xe6, 0x5d, 0x18, 0xb8,
0x24, 0x61, 0x5e, 0x28, 0xae, 0x38, 0xb6, 0x36, 0x5f, 0x88, 0x03, 0xa2, 0x0c, 0xb5, 0x9c, 0x99,
0xbf, 0x25, 0xa7, 0xf7, 0x70, 0x40, 0xd0, 0x35, 0x58, 0x3c, 0x26, 0x24, 0xb2, 0x7d, 0xea, 0x60,
0xdf, 0xd6, 0xd1, 0xa3, 0x2a, 0x80, 0x05, 0x3e, 0xf5, 0x11, 0x9f, 0xd9, 0x95, 0x11, 0x64, 0x26,
0xf0, 0xd2, 0x29, 0x9a, 0xa8, 0x0c, 0x72, 0x19, 0xba, 0x51, 0x4c, 0x1d, 0x92, 0x70, 0xef, 0x6a,
0x89, 0x84, 0x3e, 0x06, 0xa0, 0x1b, 0xb0, 0x98, 0x7e, 0x7c, 0x4c, 0x62, 0x87, 0xdf, 0xba, 0x1f,
0xc9, 0xf7, 0xb4, 0x73, 0x56, 0xd5, 0x94, 0xf9, 0xaf, 0x3e, 0xf4, 0x3f, 0x19, 0x92, 0x78, 0x94,
0x79, 0x7c, 0x4a, 0x88, 0xd2, 0x5d, 0xbf, 0x9e, 0x66, 0x20, 0x3c, 0x0f, 0x1c, 0xc6, 0x34, 0xb0,
0xd3, 0x07, 0xd6, 0x73, 0x02, 0xa5, 0xc7, 0x81, 0x77, 0xe4, 0x23, 0x2b, 0xfa, 0x00, 0xce, 0x1f,
0x7a, 0x3e, 0x23, 0xf2, 0x49, 0xb3, 0xb7, 0xfd, 0x6a, 0x79, 0x27, 0xb3, 0x32, 0xb7, 0xee, 0x08,
0x64, 0x4b, 0x11, 0xa1, 0x03, 0x58, 0xf4, 0xc2, 0x48, 0x14, 0xd4, 0xb1, 0x87, 0x7d, 0xef, 0xe9,
0xf8, 0xf9, 0xa4, 0xb7, 0xfd, 0xd6, 0x04, 0x5e, 0x77, 0x39, 0xe5, 0x7e, 0x96, 0xd0, 0x42, 0x5e,
0x09, 0x86, 0x08, 0x2c, 0xd1, 0x21, 0x2b, 0x0b, 0x99, 0x11, 0x42, 0xb6, 0x27, 0x08, 0xb9, 0x2f,
0x48, 0xf3, 0x52, 0x16, 0x69, 0x19, 0x68, 0xec, 0xc1, 0x79, 0xa9, 0x1c, 0x8f, 0xe0, 0x43, 0x8f,
0xf8, 0xfa, 0x51, 0x58, 0x7e, 0xf0, 0x53, 0x8a, 0x46, 0x24, 0xc6, 0xa1, 0x0e, 0x46, 0xfd, 0xc9,
0xf1, 0x4f, 0xb0, 0x3f, 0xd4, 0x3e, 0x26, 0x3f, 0x8c, 0xbf, 0xce, 0x00, 0x2a, 0x6b, 0xa8, 0xdf,
0x84, 0x62, 0x92, 0xf0, 0xbc, 0x99, 0x8d, 0xfe, 0xf9, 0x0c, 0x5c, 0x64, 0x80, 0x1f, 0x43, 0xd7,
0x49, 0x4e, 0x6c, 0x61, 0x12, 0x21, 0xb3, 0xb7, 0xfd, 0xde, 0x99, 0x4d, 0xba, 0xb5, 0xb3, 0xff,
0x50, 0x40, 0xad, 0x8e, 0x93, 0x9c, 0x88, 0x11, 0xfa, 0x29, 0xc0, 0xcf, 0x13, 0x1a, 0x2a, 0xce,
0x72, 0xe3, 0xdf, 0x3f, 0x3b, 0xe7, 0x1f, 0xed, 0xdf, 0xdf, 0x93, 0xac, 0xbb, 0x9c, 0x9d, 0xe4,
0xed, 0xc0, 0x5c, 0x84, 0xe3, 0xc7, 0x43, 0xc2, 0x14, 0x7b, 0xe9, 0x0b, 0xdf, 0x3d, 0x3b, 0xfb,
0x8f, 0x25, 0x1b, 0x29, 0xa1, 0x1f, 0x65, 0xbe, 0x8c, 0xaf, 0xcf, 0x41, 0x47, 0xeb, 0xc5, 0x6b,
0x72, 0xe1, 0xe1, 0xf2, 0x66, 0x6a, 0x7b, 0xe1, 0x21, 0x55, 0x16, 0xbd, 0xc0, 0xe1, 0xf2, 0x72,
0x2a, 0xb2, 0xd6, 0x26, 0x2c, 0xc4, 0xc4, 0xa1, 0xb1, 0xcb, 0x2b, 0x20, 0x2f, 0xf0, 0xb8, 0xdb,
0xcb, 0xbd, 0x9c, 0x97, 0xf0, 0x5d, 0x0d, 0x46, 0xaf, 0xc3, 0xbc, 0xd8, 0xf6, 0x0c, 0x66, 0x5b,
0xf3, 0x24, 0x7e, 0x06, 0x71, 0x13, 0x16, 0x1e, 0x0f, 0x29, 0x23, 0xb6, 0x73, 0x84, 0x63, 0xec,
0x30, 0x9a, 0xde, 0x11, 0xe7, 0x05, 0x7c, 0x27, 0x05, 0xa3, 0xff, 0x83, 0x65, 0x89, 0x4a, 0x12,
0x07, 0x47, 0x29, 0x05, 0x89, 0xd5, 0x15, 0x62, 0x49, 0xcc, 0xde, 0x16, 0x93, 0x3b, 0x7a, 0x0e,
0x19, 0xd0, 0x71, 0x68, 0x10, 0x90, 0x90, 0x25, 0x22, 0x39, 0x77, 0xad, 0xf4, 0x1b, 0xdd, 0x84,
0x55, 0xec, 0xfb, 0xf4, 0x4b, 0x5b, 0x50, 0xba, 0x76, 0x49, 0xbb, 0x59, 0x91, 0xc0, 0x0c, 0x81,
0xf4, 0x89, 0xc0, 0xb1, 0xf2, 0x8a, 0x1a, 0x57, 0xa0, 0x9b, 0xee, 0x23, 0x3f, 0xcf, 0x32, 0x0e,
0x29, 0xc6, 0xc6, 0x05, 0xe8, 0x67, 0x77, 0xc2, 0xf8, 0x67, 0x1b, 0x16, 0x2b, 0x82, 0x0a, 0x7d,
0x0e, 0xc0, 0xbd, 0x55, 0x86, 0x96, 0x72, 0xd7, 0xef, 0x9c, 0x3d, 0x38, 0xb9, 0xbf, 0x4a, 0xb0,
0xc5, 0xbd, 0x5f, 0x0e, 0xd1, 0xcf, 0xa0, 0x27, 0x3c, 0x56, 0x71, 0x97, 0x2e, 0xfb, 0xc1, 0x37,
0xe0, 0xce, 0x75, 0x55, 0xec, 0x45, 0x0c, 0xc8, 0xb1, 0xf1, 0xf7, 0x16, 0x74, 0x53, 0xc1, 0xfc,
0x74, 0x96, 0x1b, 0x25, 0xf6, 0x3a, 0xd1, 0xa7, 0xb3, 0x80, 0xdd, 0x11, 0xa0, 0xff, 0x49, 0x57,
0x32, 0xde, 0x01, 0x18, 0xeb, 0x5f, 0xa9, 0x42, 0xab, 0x52, 0x05, 0x73, 0x13, 0xe6, 0xb8, 0x65,
0x3d, 0xe2, 0xee, 0xb3, 0xd8, 0x8b, 0x44, 0x7b, 0x4b, 0xe2, 0x24, 0xea, 0x7a, 0xa1, 0x3f, 0xb7,
0xff, 0x30, 0x80, 0x7e, 0xf6, 0x59, 0x04, 0x7d, 0x01, 0xbd, 0x4c, 0x1b, 0x0f, 0xbd, 0x52, 0xde,
0xb4, 0x72, 0x5b, 0xd0, 0x78, 0x75, 0x02, 0x96, 0xba, 0x01, 0x4c, 0xa1, 0x10, 0x2e, 0x96, 0x7a,
0x61, 0xe8, 0x6a, 0x99, 0xba, 0xae, 0xd3, 0x66, 0xbc, 0xd1, 0x08, 0x37, 0x95, 0xc7, 0x60, 0xb1,
0xa2, 0xb9, 0x85, 0xde, 0x9c, 0xc0, 0x25, 0xd7, 0x60, 0x33, 0xae, 0x35, 0xc4, 0x4e, 0xa5, 0x3e,
0x06, 0x54, 0xee, 0x7c, 0xa1, 0x37, 0x26, 0xb2, 0x19, 0x77, 0xd6, 0x8c, 0x37, 0x9b, 0x21, 0xd7,
0x2a, 0x2a, 0x7b, 0x62, 0x13, 0x15, 0xcd, 0x75, 0xdd, 0x26, 0x2a, 0x5a, 0x68, 0xb4, 0x4d, 0xa1,
0x63, 0x58, 0x28, 0xf6, 0xcb, 0xd0, 0x66, 0x5d, 0x7f, 0xb7, 0xd4, 0x8e, 0x33, 0xae, 0x36, 0x41,
0x4d, 0x85, 0x11, 0xb8, 0x90, 0xef, 0x4f, 0xa1, 0xd7, 0xcb, 0xf4, 0x95, 0x1d, 0x3a, 0x63, 0x63,
0x32, 0x62, 0x56, 0xa7, 0x62, 0xcf, 0xaa, 0x4a, 0xa7, 0x9a, 0x86, 0x58, 0x95, 0x4e, 0x75, 0x2d,
0x30, 0x73, 0x0a, 0xfd, 0x42, 0x37, 0x42, 0x0a, 0xbd, 0x1c, 0xb4, 0x55, 0xc7, 0xa6, 0xba, 0x99,
0x64, 0x5c, 0x6f, 0x8c, 0xaf, 0x65, 0xdf, 0x68, 0xf1, 0x58, 0xcf, 0xb4, 0x74, 0xaa, 0x62, 0xbd,
0xdc, 0x24, 0xaa, 0x8a, 0xf5, 0xaa, 0xbe, 0xd0, 0x14, 0x3a, 0x80, 0xb9, 0x5c, 0x93, 0x07, 0xbd,
0x56, 0x47, 0x99, 0x7f, 0x1d, 0x32, 0x5e, 0x9f, 0x88, 0x97, 0xca, 0xb0, 0x75, 0xf6, 0x52, 0xe9,
0xaa, 0x76, 0x71, 0xf9, 0x7c, 0xf5, 0xda, 0x24, 0xb4, 0x5c, 0x28, 0x97, 0x5a, 0x41, 0x95, 0xa1,
0x5c, 0xd7, 0x6a, 0xaa, 0x0c, 0xe5, 0xfa, 0xee, 0xd2, 0x14, 0xfa, 0x09, 0xc0, 0xb8, 0x5d, 0x83,
0x5e, 0xae, 0xa3, 0xce, 0xee, 0xfe, 0x2b, 0xa7, 0x23, 0xa5, 0xac, 0xbf, 0x84, 0xa5, 0xaa, 0x57,
0x14, 0x74, 0xad, 0xea, 0x42, 0x58, 0xfb, 0x54, 0x63, 0x6c, 0x35, 0x45, 0x4f, 0x05, 0x7f, 0x0a,
0x1d, 0xdd, 0x6a, 0x41, 0x2f, 0x95, 0xa9, 0x0b, 0xcd, 0x25, 0xc3, 0x3c, 0x0d, 0x25, 0xe3, 0xc0,
0x81, 0x8e, 0xd5, 0x71, 0x0f, 0xa4, 0x3e, 0x56, 0x4b, 0xdd, 0x9a, 0xfa, 0x58, 0x2d, 0xb7, 0x54,
0x84, 0xb8, 0xd4, 0x19, 0xb2, 0x2d, 0x83, 0x7a, 0x67, 0xa8, 0xe8, 0x88, 0xd4, 0x3b, 0x43, 0x65,
0x17, 0x62, 0x0a, 0xfd, 0x12, 0x96, 0xab, 0x3b, 0x05, 0xa8, 0x36, 0xe2, 0x6b, 0x3a, 0x16, 0xc6,
0x8d, 0xe6, 0x04, 0xa9, 0xf8, 0xa7, 0x3a, 0x3f, 0x15, 0x3a, 0x05, 0xf5, 0xf9, 0xa9, 0xba, 0x5f,
0x61, 0x5c, 0x6f, 0x8c, 0x5f, 0x0e, 0xbd, 0xec, 0x93, 0x7c, 0xbd, 0xb5, 0x2b, 0xba, 0x0f, 0xf5,
0xd6, 0xae, 0x7c, 0xe5, 0x17, 0xf1, 0x51, 0xf5, 0xdc, 0x5e, 0x15, 0x1f, 0xa7, 0xf4, 0x03, 0x8c,
0xad, 0xa6, 0xe8, 0xb9, 0xe3, 0xbb, 0xfc, 0x9e, 0x8e, 0x26, 0xae, 0x3f, 0x97, 0x99, 0xaf, 0x35,
0xc4, 0xae, 0xdf, 0x5d, 0x9d, 0xa9, 0x27, 0x2a, 0x50, 0xc8, 0xd8, 0xd7, 0x1b, 0xe3, 0xa7, 0xb2,
0x23, 0xdd, 0x44, 0xcf, 0xbc, 0x85, 0xa3, 0xab, 0x13, 0xf8, 0x64, 0xde, 0xf2, 0x8d, 0x37, 0x1a,
0xe1, 0x56, 0x45, 0x6f, 0xf6, 0x75, 0xfa, 0x34, 0x7f, 0x2a, 0x3d, 0xa9, 0x9f, 0xe6, 0x4f, 0x15,
0x0f, 0xde, 0x53, 0xe8, 0xd7, 0xe3, 0xee, 0x68, 0xf9, 0xe5, 0x09, 0x6d, 0xd7, 0xe6, 0x82, 0xda,
0x07, 0x37, 0xe3, 0xed, 0x33, 0xd1, 0x64, 0xb4, 0xff, 0x08, 0x66, 0xc4, 0x6d, 0x0b, 0xad, 0x9d,
0x7e, 0x0d, 0x33, 0xae, 0x54, 0xcf, 0xa7, 0x97, 0x09, 0xce, 0xed, 0xe0, 0xbc, 0xf8, 0x6d, 0xf0,
0xed, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x85, 0xda, 0x2e, 0x4d, 0x28, 0x00, 0x00,
}

5
weed/pb/volume_server_pb/volume_server_helper.go Normal file
View File

@ -0,0 +1,5 @@
package volume_server_pb
func (m *RemoteFile) BackendName() string {
return m.BackendType + "." + m.BackendId
}

7
weed/server/common.go
View File

@ -11,15 +11,12 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"google.golang.org/grpc"
_ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
_ "github.com/chrislusf/seaweedfs/weed/statik"
"github.com/gorilla/mux"

75
weed/server/volume_grpc_tier.go
View File

@ -1,9 +1,9 @@
package weed_server
import (
"context"
"fmt"
"os"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
@ -11,28 +11,83 @@ import (
)
// VolumeTierCopyDatToRemote copy dat file to a remote tier
func (vs *VolumeServer) VolumeTierCopyDatToRemote(ctx context.Context, req *volume_server_pb.VolumeTierCopyDatToRemoteRequest) (*volume_server_pb.VolumeTierCopyDatToRemoteResponse, error) {
func (vs *VolumeServer) VolumeTierCopyDatToRemote(req *volume_server_pb.VolumeTierCopyDatToRemoteRequest, stream volume_server_pb.VolumeServer_VolumeTierCopyDatToRemoteServer) error {
// find existing volume
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return nil, fmt.Errorf("volume %d not found", req.VolumeId)
return fmt.Errorf("volume %d not found", req.VolumeId)
}
// verify the collection
if v.Collection != req.Collection {
return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection)
return fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection)
}
// locate the disk file
diskFile, ok := v.DataBackend.(*backend.DiskFile)
if !ok {
return nil, fmt.Errorf("volume %d is not on local disk", req.VolumeId)
return fmt.Errorf("volume %d is not on local disk", req.VolumeId)
}
err := uploadFileToRemote(ctx, req, diskFile.File)
return &volume_server_pb.VolumeTierCopyDatToRemoteResponse{}, err
}
// check valid storage backend type
backendStorage, found := backend.BackendStorages[req.DestinationBackendName]
if !found {
var keys []string
for key := range backend.BackendStorages {
keys = append(keys, key)
}
return fmt.Errorf("destination %s not found, suppported: %v", req.DestinationBackendName, keys)
}
func uploadFileToRemote(ctx context.Context, req *volume_server_pb.VolumeTierCopyDatToRemoteRequest, f *os.File) error {
println("copying dat file of", f.Name(), "to remote")
// check whether the existing backend storage is the same as requested
// if same, skip
backendType, backendId := backend.BackendNameToTypeId(req.DestinationBackendName)
for _, remoteFile := range v.GetVolumeTierInfo().GetFiles() {
if remoteFile.BackendType == backendType && remoteFile.BackendId == backendId {
return fmt.Errorf("destination %s already exists", req.DestinationBackendName)
}
}
startTime := time.Now()
fn := func(progressed int64, percentage float32) error {
now := time.Now()
if now.Sub(startTime) < time.Second {
return nil
}
startTime = now
return stream.Send(&volume_server_pb.VolumeTierCopyDatToRemoteResponse{
Processed: progressed,
ProcessedPercentage: percentage,
})
}
// copy the data file
key, size, err := backendStorage.CopyFile(diskFile.File, fn)
if err != nil {
return fmt.Errorf("backend %s copy file %s: %v", req.DestinationBackendName, diskFile.String(), err)
}
// save the remote file to volume tier info
v.GetVolumeTierInfo().Files = append(v.GetVolumeTierInfo().GetFiles(), &volume_server_pb.RemoteFile{
BackendType: backendType,
BackendId: backendId,
Key: key,
Offset: 0,
FileSize: uint64(size),
ModifiedTime: uint64(time.Now().Unix()),
})
if err := v.SaveVolumeTierInfo(); err != nil {
return fmt.Errorf("volume %d fail to save remote file info: %v", v.Id, err)
}
if err := v.LoadRemoteFile(); err != nil {
return fmt.Errorf("volume %d fail to load remote file: %v", v.Id, err)
}
if !req.KeepLocalDatFile {
os.Remove(v.FileName() + ".dat")
}
return nil
}

68
weed/shell/command_volume_tier.go
View File

@ -25,14 +25,30 @@ func (c *commandVolumeTier) Name() string {
}
func (c *commandVolumeTier) Help() string {
return `copy the dat file of a volume to a remote tier
return `move the dat file of a volume to a remote tier
ec.encode [-collection=""] [-fullPercent=95] [-quietFor=1h]
ec.encode [-collection=""] [-volumeId=<volume_id>]
volume.tier [-collection=""] [-fullPercent=95] [-quietFor=1h]
volume.tier [-collection=""] -volumeId=<volume_id> -dest=<storage_backend> [-keepLocalDatFile]
This command will:
1. freeze one volume
2. copy the dat file of a volume to a remote tier
e.g.:
volume.tier -volumeId=7 -dest=s3
volume.tier -volumeId=7 -dest=s3.default
The <storage_backend> is defined in master.toml.
For example, "s3.default" in [storage.backend.s3.default]
This command will move the dat file of a volume to a remote tier.
SeaweedFS enables scalable and fast local access to lots of files,
and the cloud storage is slower by cost efficient. How to combine them together?
Usually the data follows 80/20 rule: only 20% of data is frequently accessed.
We can offload the old volumes to the cloud.
With this, SeaweedFS can be both fast and scalable, and infinite storage space.
Just add more local SeaweedFS volume servers to increase the throughput.
The index file is still local, and the same O(1) disk read is applied to the remote file.
`
}
@ -44,7 +60,8 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io.
collection := tierCommand.String("collection", "", "the collection name")
fullPercentage := tierCommand.Float64("fullPercent", 95, "the volume reaches the percentage of max volume size")
quietPeriod := tierCommand.Duration("quietFor", 24*time.Hour, "select volumes without no writes for this period")
dest := tierCommand.String("destination", "", "the target tier name")
dest := tierCommand.String("dest", "", "the target tier name")
keepLocalDatFile := tierCommand.Bool("keepLocalDatFile", false, "whether keep local dat file")
if err = tierCommand.Parse(args); err != nil {
return nil
}
@ -54,7 +71,7 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io.
// volumeId is provided
if vid != 0 {
return doVolumeTier(ctx, commandEnv, *collection, vid, *dest)
return doVolumeTier(ctx, commandEnv, writer, *collection, vid, *dest, *keepLocalDatFile)
}
// apply to all volumes in the collection
@ -65,7 +82,7 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io.
}
fmt.Printf("tiering volumes: %v\n", volumeIds)
for _, vid := range volumeIds {
if err = doVolumeTier(ctx, commandEnv, *collection, vid, *dest); err != nil {
if err = doVolumeTier(ctx, commandEnv, writer, *collection, vid, *dest, *keepLocalDatFile); err != nil {
return err
}
}
@ -73,7 +90,7 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io.
return nil
}
func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string, vid needle.VolumeId, dest string) (err error) {
func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, writer io.Writer, collection string, vid needle.VolumeId, dest string, keepLocalDatFile bool) (err error) {
// find volume location
locations, found := commandEnv.MasterClient.GetLocations(uint32(vid))
if !found {
@ -89,7 +106,7 @@ func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string
*/
// copy the .dat file to remote tier
err = copyDatToRemoteTier(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(vid), collection, locations[0].Url, dest)
err = copyDatToRemoteTier(ctx, commandEnv.option.GrpcDialOption, writer, needle.VolumeId(vid), collection, locations[0].Url, dest, keepLocalDatFile)
if err != nil {
return fmt.Errorf("copy dat file for volume %d on %s to %s: %v", vid, locations[0].Url, dest, err)
}
@ -97,13 +114,34 @@ func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string
return nil
}
func copyDatToRemoteTier(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer string, dest string) error {
func copyDatToRemoteTier(ctx context.Context, grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, collection string, sourceVolumeServer string, dest string, keepLocalDatFile bool) error {
err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, copyErr := volumeServerClient.VolumeTierCopyDatToRemote(ctx, &volume_server_pb.VolumeTierCopyDatToRemoteRequest{
VolumeId: uint32(volumeId),
Collection: collection,
stream, copyErr := volumeServerClient.VolumeTierCopyDatToRemote(ctx, &volume_server_pb.VolumeTierCopyDatToRemoteRequest{
VolumeId: uint32(volumeId),
Collection: collection,
DestinationBackendName: dest,
KeepLocalDatFile: keepLocalDatFile,
})
var lastProcessed int64
for {
resp, recvErr := stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
} else {
return recvErr
}
}
processingSpeed := float64(resp.Processed - lastProcessed)/1024.0/1024.0
fmt.Fprintf(writer, "copied %.2f%%, %d bytes, %.2fMB/s\n", resp.ProcessedPercentage, resp.Processed, processingSpeed)
lastProcessed = resp.Processed
}
return copyErr
})

27
weed/storage/backend/backend.go
View File

@ -8,6 +8,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/spf13/viper"
)
@ -23,7 +24,8 @@ type BackendStorageFile interface {
type BackendStorage interface {
ToProperties() map[string]string
NewStorageFile(key string) BackendStorageFile
NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) BackendStorageFile
CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
}
type StringProperties interface {
@ -46,13 +48,13 @@ func LoadConfiguration(config *viper.Viper) {
backendSub := config.Sub(StorageBackendPrefix)
for backendTypeName, _ := range config.GetStringMap(StorageBackendPrefix) {
for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
if !found {
glog.Fatalf("backend storage type %s not found", backendTypeName)
}
backendTypeSub := backendSub.Sub(backendTypeName)
for backendStorageId, _ := range backendSub.GetStringMap(backendTypeName) {
for backendStorageId := range backendSub.GetStringMap(backendTypeName) {
if !backendTypeSub.GetBool(backendStorageId + ".enabled") {
continue
}
@ -105,12 +107,10 @@ func (p *Properties) GetString(key string) string {
func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
for sName, s := range BackendStorages {
parts := strings.Split(sName, ".")
if len(parts) != 2 {
sType, sId := BackendNameToTypeId(sName)
if sType == "" {
continue
}
sType, sId := parts[0], parts[1]
backends = append(backends, &master_pb.StorageBackend{
Type: sType,
Id: sId,
@ -119,3 +119,16 @@ func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
}
return
}
func BackendNameToTypeId(backendName string) (backendType, backendId string) {
parts := strings.Split(backendName, ".")
if len(parts) == 1 {
return backendName, "default"
}
if len(parts) != 2 {
return
}
backendType, backendId = parts[0], parts[1]
return
}

56
weed/storage/backend/s3_backend/s3_backend.go
View File

@ -2,6 +2,7 @@ package s3_backend
import (
"fmt"
"io"
"os"
"strings"
"time"
@ -9,7 +10,9 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
"github.com/google/uuid"
)
func init() {
@ -57,7 +60,7 @@ func (s *S3BackendStorage) ToProperties() map[string]string {
return m
}
func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile {
func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) backend.BackendStorageFile {
if strings.HasPrefix(key, "/") {
key = key[1:]
}
@ -65,18 +68,35 @@ func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile
f := &S3BackendStorageFile{
backendStorage: s,
key: key,
tierInfo: tierInfo,
}
return f
}
func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) {
randomUuid, _ := uuid.NewRandom()
key = randomUuid.String()
glog.V(1).Infof("copying dat file of", f.Name(), "to remote s3", s.id, "as", key)
size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, fn)
return
}
type S3BackendStorageFile struct {
backendStorage *S3BackendStorage
key string
tierInfo *volume_server_pb.VolumeTierInfo
}
func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {
bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1)
// glog.V(0).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{
Bucket: &s3backendStorageFile.backendStorage.bucket,
Key: &s3backendStorageFile.key,
@ -84,13 +104,26 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n
})
if getObjectErr != nil {
return 0, fmt.Errorf("bucket %s GetObject %s: %v",
s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
}
defer getObjectOutput.Body.Close()
return getObjectOutput.Body.Read(p)
glog.V(4).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
glog.V(4).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength)
for {
if n, err = getObjectOutput.Body.Read(p); err == nil && n < len(p) {
p = p[n:]
} else {
break
}
}
if err == io.EOF {
err = nil
}
return
}
func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
@ -107,18 +140,15 @@ func (s3backendStorageFile S3BackendStorageFile) Close() error {
func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {
headObjectOutput, headObjectErr := s3backendStorageFile.backendStorage.conn.HeadObject(&s3.HeadObjectInput{
Bucket: &s3backendStorageFile.backendStorage.bucket,
Key: &s3backendStorageFile.key,
})
files := s3backendStorageFile.tierInfo.GetFiles()
if headObjectErr != nil {
return 0, time.Now(), fmt.Errorf("bucket %s HeadObject %s: %v",
s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, headObjectErr)
if len(files)==0 {
err = fmt.Errorf("remote file info not found")
return
}
datSize = int64(*headObjectOutput.ContentLength)
modTime = *headObjectOutput.LastModified
datSize = int64(files[0].FileSize)
modTime = time.Unix(int64(files[0].ModifiedTime),0)
return
}

66
weed/storage/backend/s3_backend/s3_upload.go
View File

@ -3,27 +3,30 @@ package s3_backend
import (
"fmt"
"os"
"sync/atomic"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string) error {
func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string,
fn func(progressed int64, percentage float32)error) (fileSize int64, err error) {
//open the file
f, err := os.Open(filename)
if err != nil {
return fmt.Errorf("failed to open file %q, %v", filename, err)
return 0, fmt.Errorf("failed to open file %q, %v", filename, err)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return fmt.Errorf("failed to stat file %q, %v", filename, err)
return 0, fmt.Errorf("failed to stat file %q, %v", filename, err)
}
fileSize := info.Size()
fileSize = info.Size()
partSize := int64(64 * 1024 * 1024) // The minimum/default allowed part size is 5MB
for partSize*1000 < fileSize {
@ -33,14 +36,22 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey
// Create an uploader with the session and custom options
uploader := s3manager.NewUploaderWithClient(sess, func(u *s3manager.Uploader) {
u.PartSize = partSize
u.Concurrency = 15 // default is 15
u.Concurrency = 5
})
fileReader := &s3UploadProgressedReader{
fp: f,
size:fileSize,
read:-fileSize,
fn:fn,
}
// Upload the file to S3.
result, err := uploader.Upload(&s3manager.UploadInput{
var result *s3manager.UploadOutput
result, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(destBucket),
Key: aws.String(destKey),
Body: f,
Body: fileReader,
ACL: aws.String("private"),
ServerSideEncryption: aws.String("AES256"),
StorageClass: aws.String("STANDARD_IA"),
@ -48,9 +59,44 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey
//in case it fails to upload
if err != nil {
return fmt.Errorf("failed to upload file, %v", err)
return 0, fmt.Errorf("failed to upload file %s: %v", filename, err)
}
fmt.Printf("file %s uploaded to %s\n", filename, result.Location)
glog.V(1).Infof("file %s uploaded to %s\n", filename, result.Location)
return nil
return
}
// adapted from https://github.com/aws/aws-sdk-go/pull/1868
type s3UploadProgressedReader struct {
fp *os.File
size int64
read int64
fn func(progressed int64, percentage float32)error
}
func (r *s3UploadProgressedReader) Read(p []byte) (int, error) {
return r.fp.Read(p)
}
func (r *s3UploadProgressedReader) ReadAt(p []byte, off int64) (int, error) {
n, err := r.fp.ReadAt(p, off)
if err != nil {
return n, err
}
// Got the length have read( or means has uploaded), and you can construct your message
atomic.AddInt64(&r.read, int64(n))
if r.fn != nil {
read := r.read
if err := r.fn(read, float32(read*100)/float32(r.size)); err != nil {
return n, err
}
}
return n, err
}
func (r *s3UploadProgressedReader) Seek(offset int64, whence int) (int64, error) {
return r.fp.Seek(offset, whence)
}

6
weed/storage/disk_location.go
View File

@ -33,8 +33,8 @@ func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
base := name[:len(name)-len(".dat")]
if !dir.IsDir() && strings.HasSuffix(name, ".idx") {
base := name[:len(name)-len(".idx")]
collection, volumeId, err := parseCollectionVolumeId(base)
return volumeId, collection, err
}
@ -53,7 +53,7 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI
func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) {
name := fileInfo.Name()
if !fileInfo.IsDir() && strings.HasSuffix(name, ".dat") {
if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") {
vid, collection, err := l.volumeIdFromPath(fileInfo)
if err == nil {
l.RLock()

3
weed/storage/volume.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
@ -37,6 +38,8 @@ type Volume struct {
lastCompactRevision uint16
isCompacting bool
volumeTierInfo *volume_server_pb.VolumeTierInfo
}
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) {

7
weed/storage/volume_loading.go
View File

@ -26,8 +26,11 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
fileName := v.FileName()
alreadyHasSuperBlock := false
// open dat file
if exists, canRead, canWrite, modifiedTime, fileSize := checkFile(fileName + ".dat"); exists {
if v.maybeLoadVolumeTierInfo() {
// open remote file
alreadyHasSuperBlock = true
} else if exists, canRead, canWrite, modifiedTime, fileSize := checkFile(fileName + ".dat"); exists {
// open dat file
if !canRead {
return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
}

104
weed/storage/volume_tier.go Normal file
View File

@ -0,0 +1,104 @@
package storage
import (
"bytes"
"fmt"
"io/ioutil"
_ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
"github.com/golang/protobuf/jsonpb"
)
func (v *Volume) GetVolumeTierInfo() *volume_server_pb.VolumeTierInfo {
return v.volumeTierInfo
}
func (v *Volume) maybeLoadVolumeTierInfo() bool {
v.volumeTierInfo = &volume_server_pb.VolumeTierInfo{}
tierFileName := v.FileName() + ".tier"
if exists, canRead, _, _, _ := checkFile(tierFileName); !exists || !canRead {
if !exists {
return false
}
if !canRead {
glog.Warningf("can not read %s", tierFileName)
}
return false
}
glog.V(0).Infof("maybeLoadVolumeTierInfo loading volume %d check file", v.Id)
tierData, readErr := ioutil.ReadFile(tierFileName)
if readErr != nil {
glog.Warningf("fail to read %s : %v", tierFileName, readErr)
return false
}
glog.V(0).Infof("maybeLoadVolumeTierInfo loading volume %d ReadFile", v.Id)
if err := jsonpb.Unmarshal(bytes.NewReader(tierData), v.volumeTierInfo); err != nil {
glog.Warningf("unmarshal error: %v", err)
return false
}
glog.V(0).Infof("maybeLoadVolumeTierInfo loading volume %d Unmarshal tierInfo %v", v.Id, v.volumeTierInfo)
if len(v.volumeTierInfo.GetFiles()) == 0 {
return false
}
glog.V(0).Infof("volume %d is tiered to %s as %s and read only", v.Id,
v.volumeTierInfo.Files[0].BackendName(), v.volumeTierInfo.Files[0].Key)
v.readOnly = true
glog.V(0).Infof("loading volume %d from remote %v", v.Id, v.volumeTierInfo.Files)
v.LoadRemoteFile()
return true
}
func (v *Volume) LoadRemoteFile() error {
tierFile := v.volumeTierInfo.GetFiles()[0]
backendStorage := backend.BackendStorages[tierFile.BackendName()]
if v.DataBackend != nil {
v.DataBackend.Close()
}
v.DataBackend = backendStorage.NewStorageFile(tierFile.Key, v.volumeTierInfo)
return nil
}
func (v *Volume) SaveVolumeTierInfo() error {
tierFileName := v.FileName() + ".tier"
if exists, _, canWrite, _, _ := checkFile(tierFileName); exists && !canWrite {
return fmt.Errorf("%s not writable", tierFileName)
}
m := jsonpb.Marshaler{
EmitDefaults: true,
Indent: " ",
}
text, marshalErr := m.MarshalToString(v.GetVolumeTierInfo())
if marshalErr != nil {
return fmt.Errorf("marshal volume %d tier info: %v", v.Id, marshalErr)
}
writeErr := ioutil.WriteFile(tierFileName, []byte(text), 0755)
if writeErr != nil {
return fmt.Errorf("fail to write %s : %v", tierFileName, writeErr)
}
return nil
}