lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-04-05 20:52:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

[master] process grow request with must grow (#5999)

Browse Source 
process grow request with must grow
This commit is contained in:
Konstantin Lebedev 2024-09-10 11:45:02 +05:00 committed by GitHub
parent 15965f7c54
commit 34bbaa2cdd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 43 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
weed/server/master_grpc_server_volume.go
View File

@ -3,6 +3,7 @@ package weed_server
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/stats"
"math/rand/v2"
"strings"
"sync"
@ -20,6 +21,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
const (
volumeGrowStepCount = 2
)
func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
glog.V(1).Infoln("starting automatic volume grow")
start := time.Now()
@ -36,38 +41,48 @@ func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
func (ms *MasterServer) ProcessGrowRequest() {
go func() {
ctx := context.Background()
for {
if !ms.Topo.IsLeader() {
continue
}
dcs := ms.Topo.ListDataCenters()
var err error
for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
vl := vlc.VolumeLayout
lastGrowCount := vl.GetLastGrowCount()
if vl.HasGrowRequest() {
continue
}
if vl.ShouldGrowVolumes(vlc.Collection) {
vl.AddGrowRequest()
ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
Option: vlc.ToGrowOption(),
Count: vl.GetLastGrowCount(),
Reason: "collection autogrow",
}
} else {
writable, crowded := vl.GetWritableVolumeCount()
mustGrow := int(lastGrowCount) - writable
vgr := vlc.ToVolumeGrowRequest()
stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable))
stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded))
switch {
case mustGrow > 0:
vgr.WritableVolumeCount = uint32(mustGrow)
_, err = ms.VolumeGrow(ctx, vgr)
case crowded+volumeGrowStepCount >= writable:
vgr.WritableVolumeCount = volumeGrowStepCount
_, err = ms.VolumeGrow(ctx, vgr)
default:
for _, dc := range dcs {
if vl.ShouldGrowVolumesByDataNode("DataCenter", dc) {
vl.AddGrowRequest()
volumeGrowOption := vlc.ToGrowOption()
volumeGrowOption.DataCenter = dc
ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
Option: volumeGrowOption,
Count: vl.GetLastGrowCount(),
Force: true,
Reason: "per-dc autogrow",
vgr.DataCenter = dc
if lastGrowCount > 0 {
vgr.WritableVolumeCount = uint32(int(lastGrowCount) / len(dcs))
} else {
vgr.WritableVolumeCount = volumeGrowStepCount
}
_, err = ms.VolumeGrow(ctx, vgr)
}
}
}
if err != nil {
glog.V(0).Infof("volume grow request failed: %+v", err)
}
}
time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
}
@ -101,7 +116,7 @@ func (ms *MasterServer) ProcessGrowRequest() {
})
// not atomic but it's okay
if found || (!req.Force && !vl.ShouldGrowVolumes(req.Option.Collection)) {
if found || (!req.Force && !vl.ShouldGrowVolumes()) {
glog.V(4).Infoln("discard volume grow request")
time.Sleep(time.Millisecond * 211)
vl.DoneGrowRequest()
@ -302,6 +317,9 @@ func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGro
if err != nil {
return nil, err
}
if req.DataCenter != "" && !ms.Topo.DataCenterExists(req.DataCenter) {
return nil, fmt.Errorf("data center not exists")
}
volumeGrowOption := topology.VolumeGrowOption{
Collection: req.Collection,
ReplicaPlacement: replicaPlacement,

18
weed/topology/volume_layout.go
View File

@ -2,7 +2,7 @@ package topology
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"math/rand"
"sync"
"sync/atomic"
@ -359,10 +359,8 @@ func (vl *VolumeLayout) GetLastGrowCount() uint32 {
return vl.lastGrowCount.Load()
}
func (vl *VolumeLayout) ShouldGrowVolumes(collection string) bool {
func (vl *VolumeLayout) ShouldGrowVolumes() bool {
writable, crowded := vl.GetWritableVolumeCount()
stats.MasterVolumeLayoutWritable.WithLabelValues(collection, vl.diskType.String(), vl.rp.String(), vl.ttl.String()).Set(float64(writable))
stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, vl.diskType.String(), vl.rp.String(), vl.ttl.String()).Set(float64(crowded))
return writable <= crowded
}
@ -544,12 +542,12 @@ func (vl *VolumeLayout) ToInfo() (info VolumeLayoutInfo) {
return
}
func (vlc *VolumeLayoutCollection) ToGrowOption() (option *VolumeGrowOption) {
return &VolumeGrowOption{
Collection: vlc.Collection,
ReplicaPlacement: vlc.VolumeLayout.rp,
Ttl: vlc.VolumeLayout.ttl,
DiskType: vlc.VolumeLayout.diskType,
func (vlc *VolumeLayoutCollection) ToVolumeGrowRequest() *master_pb.VolumeGrowRequest {
return &master_pb.VolumeGrowRequest{
Collection: vlc.Collection,
Replication: vlc.VolumeLayout.rp.String(),
Ttl: vlc.VolumeLayout.ttl.String(),
DiskType: vlc.VolumeLayout.diskType.String(),
}
}