mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-04-05 20:52:50 +08:00
Merge branch 'master' into mq
This commit is contained in:
chrislu
commit
5038577f7e
38
go.mod
38
go.mod
@ -3,13 +3,13 @@ module github.com/seaweedfs/seaweedfs
|
||||
go 1.22.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.112.2 // indirect
|
||||
cloud.google.com/go/pubsub v1.37.0
|
||||
cloud.google.com/go/storage v1.40.0
|
||||
cloud.google.com/go v0.113.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.38.0
|
||||
cloud.google.com/go/storage v1.41.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.51.30
|
||||
github.com/aws/aws-sdk-go v1.53.5
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
@ -41,7 +41,7 @@ require (
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.6.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.12.3 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.12.4 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@ -106,17 +106,17 @@ require (
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.37.0
|
||||
golang.org/x/crypto v0.23.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
|
||||
golang.org/x/image v0.15.0
|
||||
golang.org/x/image v0.16.0
|
||||
golang.org/x/net v0.25.0
|
||||
golang.org/x/oauth2 v0.19.0 // indirect
|
||||
golang.org/x/oauth2 v0.20.0 // indirect
|
||||
golang.org/x/sys v0.20.0
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
golang.org/x/tools v0.21.0
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
|
||||
google.golang.org/api v0.177.0
|
||||
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
|
||||
google.golang.org/api v0.181.0
|
||||
google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect
|
||||
google.golang.org/grpc v1.63.2
|
||||
google.golang.org/protobuf v1.34.0
|
||||
google.golang.org/protobuf v1.34.1
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/libc v1.49.3 // indirect
|
||||
@ -131,10 +131,10 @@ require (
|
||||
github.com/Jille/raft-grpc-transport v1.5.0
|
||||
github.com/arangodb/go-driver v1.6.2
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.27.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
|
||||
github.com/cognusion/imaging v1.0.1
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/getsentry/sentry-go v0.27.0
|
||||
@ -149,20 +149,20 @@ require (
|
||||
github.com/rclone/rclone v1.66.0
|
||||
github.com/rdleal/intervalst v1.4.0
|
||||
github.com/schollz/progressbar/v3 v3.14.2
|
||||
github.com/shirou/gopsutil/v3 v3.24.3
|
||||
github.com/shirou/gopsutil/v3 v3.24.4
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.13
|
||||
go.uber.org/atomic v1.11.0
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.3.0 // indirect
|
||||
cloud.google.com/go/auth v0.4.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.3.0 // indirect
|
||||
cloud.google.com/go/iam v1.1.7 // indirect
|
||||
cloud.google.com/go/iam v1.1.8 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
|
||||
@ -309,7 +309,7 @@ require (
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@ -329,8 +329,8 @@ require (
|
||||
golang.org/x/sync v0.7.0 // indirect
|
||||
golang.org/x/term v0.20.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
|
||||
94
go.sum
94
go.sum
@ -14,10 +14,10 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw=
|
||||
cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms=
|
||||
cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs=
|
||||
cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w=
|
||||
cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA=
|
||||
cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8=
|
||||
cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg=
|
||||
cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
@ -30,23 +30,23 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2Qx
|
||||
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM=
|
||||
cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA=
|
||||
cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM=
|
||||
cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI=
|
||||
cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0=
|
||||
cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE=
|
||||
cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs=
|
||||
cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ=
|
||||
cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ=
|
||||
cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc=
|
||||
cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw=
|
||||
cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g=
|
||||
cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0=
|
||||
cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
@ -140,10 +140,10 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh
|
||||
github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
|
||||
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
|
||||
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
|
||||
github.com/aws/aws-sdk-go v1.51.30 h1:RVFkjn9P0JMwnuZCVH0TlV5k9zepHzlbc4943eZMhGw=
|
||||
github.com/aws/aws-sdk-go v1.51.30/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
|
||||
github.com/aws/aws-sdk-go v1.53.5 h1:1OcVWMjGlwt7EU5OWmmEEXqaYfmX581EK317QJZXItM=
|
||||
github.com/aws/aws-sdk-go v1.53.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
|
||||
@ -168,8 +168,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/g
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 h1:rq2hglTQM3yHZvOPVMtNvLS5x6hijx7JvRDgKiTNDGQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 h1:kHm1SYs/NkxZpKINc4zOXOLJHVMzKtU4d7FlAMtDm50=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.29.2/go.mod h1:ZIs7/BaYel9NODoYa8PW39o15SFAXDEb4DxOG2It15U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 h1:A9ihuyTKpS8Z1ou/D4ETfOEFMyokA6JjRsgXWTiHvCk=
|
||||
@ -464,8 +464,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
|
||||
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
||||
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
@ -490,8 +490,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA=
|
||||
github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
|
||||
github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg=
|
||||
github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
@ -844,8 +844,8 @@ github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQ
|
||||
github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ=
|
||||
github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE=
|
||||
github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg=
|
||||
github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU=
|
||||
github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
|
||||
@ -958,14 +958,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf h1:ckwNHVo4bv2tqNkgx3W3HANh3ta1j6TR5qw08J1A7Tw=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca h1:PliQWLwi2gTSOk7QyYQ9GfjvvikmibLWmaplKHy+kfo=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfGLkaDHuAEnWX63/8RBBiA4=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0 h1:JHnWtMHGttTRgIDPB5waC9WYcaEsL4x4LWfrs/zd6jQ=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0/go.mod h1:hJnZV1xFlpwNeH5Hy3N2TlLYl18OmDP5P1puEsRDNrY=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3 h1:XeY8gUh18MNJW0+TmPx8rl+vpnWwwiqgII27JdPmgJA=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3/go.mod h1:hGX4CijskNnUTRgLlqMvZdrBQc1ALZgAnKHytF5nmj4=
|
||||
github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo=
|
||||
github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
|
||||
@ -991,8 +991,8 @@ github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
|
||||
github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8=
|
||||
go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M=
|
||||
go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI=
|
||||
go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI=
|
||||
go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
|
||||
go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
|
||||
go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=
|
||||
@ -1018,8 +1018,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
|
||||
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
|
||||
go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
|
||||
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
|
||||
go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
|
||||
go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
|
||||
go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
|
||||
go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
|
||||
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
|
||||
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
@ -1088,8 +1088,8 @@ golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu
|
||||
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
|
||||
golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
|
||||
golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@ -1176,8 +1176,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg=
|
||||
golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8=
|
||||
golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
|
||||
golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -1266,7 +1266,7 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
@ -1385,8 +1385,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.177.0 h1:8a0p/BbPa65GlqGWtUKxot4p0TV8OGOfyTjtmkXNXmk=
|
||||
google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOnFQDw=
|
||||
google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4=
|
||||
google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@ -1427,12 +1427,12 @@ google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6D
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s=
|
||||
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
|
||||
google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw=
|
||||
google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@ -1475,8 +1475,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4=
|
||||
google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
|
||||
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -12,6 +12,9 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: filer
|
||||
{{- with .Values.global.monitoring.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
|
||||
@ -62,6 +62,9 @@ spec:
|
||||
initContainers:
|
||||
{{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "filer.image" . }}
|
||||
@ -84,11 +87,13 @@ spec:
|
||||
secretKeyRef:
|
||||
name: secret-seaweedfs-db
|
||||
key: user
|
||||
optional: true
|
||||
- name: WEED_MYSQL_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: secret-seaweedfs-db
|
||||
key: password
|
||||
optional: true
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ template "seaweedfs.name" . }}"
|
||||
{{- if .Values.filer.extraEnvironmentVars }}
|
||||
@ -262,6 +267,9 @@ spec:
|
||||
resources:
|
||||
{{ tpl .Values.filer.resources . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.sidecars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@ -12,6 +12,9 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: master
|
||||
{{- with .Values.global.monitoring.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
|
||||
@ -61,6 +61,9 @@ spec:
|
||||
initContainers:
|
||||
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.master.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "master.image" . }}
|
||||
@ -222,6 +225,9 @@ spec:
|
||||
resources:
|
||||
{{ tpl .Values.master.resources . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.sidecars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@ -22,6 +22,9 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
{{- if .Values.filer.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: post-install-job
|
||||
image: {{ template "master.image" . }}
|
||||
@ -80,6 +83,9 @@ spec:
|
||||
{{- end }}
|
||||
- containerPort: {{ .Values.master.grpcPort }}
|
||||
#name: swfs-master-grpc
|
||||
{{- if .Values.filer.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
volumes:
|
||||
- name: config-users
|
||||
|
||||
@ -50,6 +50,9 @@ spec:
|
||||
initContainers:
|
||||
{{ tpl .Values.s3.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.s3.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "s3.image" . }}
|
||||
@ -199,6 +202,9 @@ spec:
|
||||
resources:
|
||||
{{ tpl .Values.s3.resources . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.sidecars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@ -12,6 +12,9 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{- with .Values.global.monitoring.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
|
||||
@ -12,6 +12,9 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
{{- with .Values.global.monitoring.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
|
||||
@ -72,6 +72,9 @@ spec:
|
||||
{{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.volume.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "volume.image" . }}
|
||||
@ -237,6 +240,9 @@ spec:
|
||||
resources:
|
||||
{{ tpl .Values.volume.resources . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.volume.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.volume.sidecars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@ -25,6 +25,7 @@ global:
|
||||
enabled: false
|
||||
gatewayHost: null
|
||||
gatewayPort: null
|
||||
additionalLabels: {}
|
||||
# if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config
|
||||
enableReplication: false
|
||||
# replication type is XYZ:
|
||||
@ -166,6 +167,25 @@ master:
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
|
||||
serviceAccountName: ""
|
||||
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# allowPrivilegeEscalation: false
|
||||
containerSecurityContext: {}
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "nginx"
|
||||
@ -377,6 +397,25 @@ volume:
|
||||
|
||||
extraEnvironmentVars:
|
||||
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# allowPrivilegeEscalation: false
|
||||
containerSecurityContext: {}
|
||||
|
||||
# used to configure livenessProbe on volume-server containers
|
||||
#
|
||||
livenessProbe:
|
||||
@ -535,6 +574,25 @@ filer:
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
|
||||
serviceAccountName: ""
|
||||
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# allowPrivilegeEscalation: false
|
||||
containerSecurityContext: {}
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "nginx"
|
||||
@ -605,7 +663,7 @@ filer:
|
||||
timeoutSeconds: 10
|
||||
|
||||
# secret env variables
|
||||
secretExtraEnvironmentVars: []
|
||||
secretExtraEnvironmentVars: {}
|
||||
# WEED_POSTGRES_USERNAME:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
@ -706,6 +764,25 @@ s3:
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
|
||||
serviceAccountName: ""
|
||||
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# allowPrivilegeEscalation: false
|
||||
containerSecurityContext: {}
|
||||
|
||||
logs:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
|
||||
@ -32,12 +32,15 @@ var cmdFix = &Command{
|
||||
var (
|
||||
fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
|
||||
fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
|
||||
fixIncludeDeleted = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
|
||||
fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
|
||||
)
|
||||
|
||||
type VolumeFileScanner4Fix struct {
|
||||
version needle.Version
|
||||
nm *needle_map.MemDb
|
||||
version needle.Version
|
||||
nm *needle_map.MemDb
|
||||
nmDeleted *needle_map.MemDb
|
||||
includeDeleted bool
|
||||
}
|
||||
|
||||
func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
|
||||
@ -50,13 +53,20 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
|
||||
}
|
||||
|
||||
func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
|
||||
glog.V(2).Infof("key %d offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
|
||||
glog.V(2).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
|
||||
if n.Size.IsValid() {
|
||||
pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size)
|
||||
glog.V(2).Infof("saved %d with error %v", n.Size, pe)
|
||||
if pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
|
||||
return fmt.Errorf("saved %d with error %v", n.Size, pe)
|
||||
}
|
||||
} else {
|
||||
glog.V(2).Infof("skipping deleted file ...")
|
||||
return scanner.nm.Delete(n.Id)
|
||||
if scanner.includeDeleted {
|
||||
if pe := scanner.nmDeleted.Set(n.Id, types.ToOffset(offset), types.TombstoneFileSize); pe != nil {
|
||||
return fmt.Errorf("saved deleted %d with error %v", n.Size, pe)
|
||||
}
|
||||
} else {
|
||||
glog.V(2).Infof("skipping deleted file ...")
|
||||
return scanner.nm.Delete(n.Id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -109,21 +119,45 @@ func runFix(cmd *Command, args []string) bool {
|
||||
if *fixVolumeId != 0 && *fixVolumeId != volumeId {
|
||||
continue
|
||||
}
|
||||
doFixOneVolume(basePath, baseFileName, collection, volumeId)
|
||||
doFixOneVolume(basePath, baseFileName, collection, volumeId, *fixIncludeDeleted)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64) {
|
||||
func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
|
||||
idxFile, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
idxFile.Close()
|
||||
}()
|
||||
|
||||
return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error {
|
||||
_, err := idxFile.Write(value.ToBytes())
|
||||
if scaner.includeDeleted && err == nil {
|
||||
if deleted, ok := scaner.nmDeleted.Get(value.Key); ok {
|
||||
_, err = idxFile.Write(deleted.ToBytes())
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
|
||||
indexFileName := path.Join(basepath, baseFileName+".idx")
|
||||
|
||||
nm := needle_map.NewMemDb()
|
||||
nmDeleted := needle_map.NewMemDb()
|
||||
defer nm.Close()
|
||||
defer nmDeleted.Close()
|
||||
|
||||
vid := needle.VolumeId(volumeId)
|
||||
scanner := &VolumeFileScanner4Fix{
|
||||
nm: nm,
|
||||
nm: nm,
|
||||
nmDeleted: nmDeleted,
|
||||
includeDeleted: fixIncludeDeleted,
|
||||
}
|
||||
|
||||
if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
|
||||
@ -135,12 +169,12 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol
|
||||
}
|
||||
}
|
||||
|
||||
if err := nm.SaveToIdx(indexFileName); err != nil {
|
||||
os.Remove(indexFileName)
|
||||
if err := SaveToIdx(scanner, indexFileName); err != nil {
|
||||
err := fmt.Errorf("save to .idx File: %v", err)
|
||||
if *fixIgnoreError {
|
||||
glog.Error(err)
|
||||
} else {
|
||||
os.Remove(indexFileName)
|
||||
glog.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -280,6 +280,13 @@ tls_client_key_file=""
|
||||
[mongodb]
|
||||
enabled = false
|
||||
uri = "mongodb://localhost:27017"
|
||||
username = ""
|
||||
password = ""
|
||||
ssl = false
|
||||
ssl_ca_file = ""
|
||||
ssl_cert_file = ""
|
||||
ssl_key_file = "
|
||||
insecure_skip_verify = false
|
||||
option_pool_size = 0
|
||||
database = "seaweedfs"
|
||||
|
||||
|
||||
@ -2,7 +2,12 @@ package mongodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@ -10,7 +15,6 @@ import (
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -37,17 +41,44 @@ func (store *MongodbStore) Initialize(configuration util.Configuration, prefix s
|
||||
store.database = configuration.GetString(prefix + "database")
|
||||
store.collectionName = "filemeta"
|
||||
poolSize := configuration.GetInt(prefix + "option_pool_size")
|
||||
return store.connection(configuration.GetString(prefix+"uri"), uint64(poolSize))
|
||||
uri := configuration.GetString(prefix + "uri")
|
||||
ssl := configuration.GetBool(prefix + "ssl")
|
||||
sslCAFile := configuration.GetString(prefix + "ssl_ca_file")
|
||||
sslCertFile := configuration.GetString(prefix + "ssl_cert_file")
|
||||
sslKeyFile := configuration.GetString(prefix + "ssl_key_file")
|
||||
username := configuration.GetString(prefix + "username")
|
||||
password := configuration.GetString(prefix + "password")
|
||||
insecure_skip_verify := configuration.GetBool(prefix + "insecure_skip_verify")
|
||||
|
||||
return store.connection(uri, uint64(poolSize), ssl, sslCAFile, sslCertFile, sslKeyFile, username, password, insecure_skip_verify)
|
||||
}
|
||||
|
||||
func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {
|
||||
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, sslCAFile, sslCertFile, sslKeyFile string, username, password string, insecure bool) (err error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
opts := options.Client().ApplyURI(uri)
|
||||
|
||||
if poolSize > 0 {
|
||||
opts.SetMaxPoolSize(poolSize)
|
||||
}
|
||||
|
||||
if ssl {
|
||||
tlsConfig, err := configureTLS(sslCAFile, sslCertFile, sslKeyFile, insecure)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts.SetTLSConfig(tlsConfig)
|
||||
}
|
||||
|
||||
if username != "" && password != "" {
|
||||
creds := options.Credential{
|
||||
Username: username,
|
||||
Password: password,
|
||||
}
|
||||
opts.SetAuth(creds)
|
||||
}
|
||||
|
||||
client, err := mongo.Connect(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -55,10 +86,36 @@ func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {
|
||||
|
||||
c := client.Database(store.database).Collection(store.collectionName)
|
||||
err = store.indexUnique(c)
|
||||
|
||||
store.connect = client
|
||||
return err
|
||||
}
|
||||
|
||||
func configureTLS(caFile, certFile, keyFile string, insecure bool) (*tls.Config, error) {
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not load client key pair: %s", err)
|
||||
}
|
||||
|
||||
caCert, err := os.ReadFile(caFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read CA certificate: %s", err)
|
||||
}
|
||||
|
||||
caCertPool := x509.NewCertPool()
|
||||
if !caCertPool.AppendCertsFromPEM(caCert) {
|
||||
return nil, fmt.Errorf("failed to append CA certificate")
|
||||
}
|
||||
|
||||
tlsConfig := &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
RootCAs: caCertPool,
|
||||
InsecureSkipVerify: insecure,
|
||||
}
|
||||
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
func (store *MongodbStore) createIndex(c *mongo.Collection, index mongo.IndexModel, opts *options.CreateIndexesOptions) error {
|
||||
_, err := c.Indexes().CreateOne(context.Background(), index, opts)
|
||||
return err
|
||||
@ -93,13 +150,10 @@ func (store *MongodbStore) RollbackTransaction(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
return store.UpdateEntry(ctx, entry)
|
||||
|
||||
}
|
||||
|
||||
func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
meta, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
@ -126,7 +180,6 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry)
|
||||
}
|
||||
|
||||
func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
|
||||
|
||||
dir, name := fullpath.DirAndName()
|
||||
var data Model
|
||||
|
||||
@ -154,7 +207,6 @@ func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath
|
||||
}
|
||||
|
||||
func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
|
||||
|
||||
dir, name := fullpath.DirAndName()
|
||||
|
||||
where := bson.M{"directory": dir, "name": name}
|
||||
@ -167,7 +219,6 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa
|
||||
}
|
||||
|
||||
func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
|
||||
|
||||
where := bson.M{"directory": fullpath}
|
||||
_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)
|
||||
if err != nil {
|
||||
@ -182,7 +233,6 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dir
|
||||
}
|
||||
|
||||
func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
|
||||
|
||||
var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}}
|
||||
if includeStartFile {
|
||||
where["name"] = bson.M{
|
||||
|
||||
@ -33,6 +33,7 @@ const (
|
||||
StatementActionReadAcp = "GetBucketAcl"
|
||||
StatementActionList = "List*"
|
||||
StatementActionTagging = "Tagging*"
|
||||
StatementActionDelete = "DeleteBucket*"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -58,6 +59,8 @@ func MapToStatementAction(action string) string {
|
||||
return s3_constants.ACTION_LIST
|
||||
case StatementActionTagging:
|
||||
return s3_constants.ACTION_TAGGING
|
||||
case StatementActionDelete:
|
||||
return s3_constants.ACTION_DELETE_BUCKET
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
@ -79,6 +82,8 @@ func MapToIdentitiesAction(action string) string {
|
||||
return StatementActionList
|
||||
case s3_constants.ACTION_TAGGING:
|
||||
return StatementActionTagging
|
||||
case s3_constants.ACTION_DELETE_BUCKET:
|
||||
return StatementActionDelete
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
@ -67,7 +67,7 @@ func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64)
|
||||
maxStop = max(maxStop, logicStop)
|
||||
|
||||
if t.TsNs >= tsNs {
|
||||
println("read new data1", t.TsNs - tsNs, "ns")
|
||||
println("read new data1", t.TsNs-tsNs, "ns")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -137,7 +137,7 @@ func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop in
|
||||
maxStop = max(maxStop, logicStop)
|
||||
|
||||
if t.TsNs >= tsNs {
|
||||
println("read new data2", t.TsNs - tsNs, "ns")
|
||||
println("read new data2", t.TsNs-tsNs, "ns")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,7 +35,6 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
t := topic.FromPbTopic(request.Topic)
|
||||
var readErr, assignErr error
|
||||
resp, readErr = b.fca.ReadTopicConfFromFiler(t)
|
||||
|
||||
@ -69,7 +69,7 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
|
||||
return stream.Send(response)
|
||||
}
|
||||
|
||||
var receivedSequence, acknowledgedSequence int64
|
||||
var receivedSequence, acknowledgedSequence int64
|
||||
var isClosed bool
|
||||
|
||||
// start sending ack to publisher
|
||||
@ -85,7 +85,7 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
|
||||
lastAckTime := time.Now()
|
||||
for !isClosed {
|
||||
receivedSequence = atomic.LoadInt64(&localTopicPartition.AckTsNs)
|
||||
if acknowledgedSequence < receivedSequence && (receivedSequence - acknowledgedSequence >= ackInterval || time.Since(lastAckTime) > 1*time.Second){
|
||||
if acknowledgedSequence < receivedSequence && (receivedSequence-acknowledgedSequence >= ackInterval || time.Since(lastAckTime) > 1*time.Second) {
|
||||
acknowledgedSequence = receivedSequence
|
||||
response := &mq_pb.PublishMessageResponse{
|
||||
AckSequence: acknowledgedSequence,
|
||||
@ -101,7 +101,6 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
// process each published messages
|
||||
clientName := fmt.Sprintf("%v-%4d/%s/%v", findClientAddress(stream.Context()), rand.Intn(10000), initMessage.Topic, initMessage.Partition)
|
||||
localTopicPartition.Publishers.AddPublisher(clientName, topic.NewLocalPublisher())
|
||||
|
||||
@ -13,10 +13,11 @@ import (
|
||||
)
|
||||
|
||||
type memBuffer struct {
|
||||
buf []byte
|
||||
startTime time.Time
|
||||
stopTime time.Time
|
||||
buf []byte
|
||||
startTime time.Time
|
||||
stopTime time.Time
|
||||
}
|
||||
|
||||
func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_PublishFollowMeServer) (err error) {
|
||||
var req *mq_pb.PublishFollowMeRequest
|
||||
req, err = stream.Recv()
|
||||
@ -84,7 +85,6 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
|
||||
|
||||
logBuffer.ShutdownLogBuffer()
|
||||
@ -97,7 +97,6 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi
|
||||
partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
|
||||
partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
|
||||
|
||||
|
||||
// flush the remaining messages
|
||||
inMemoryBuffers.CloseInput()
|
||||
for mem, found := inMemoryBuffers.Dequeue(); found; mem, found = inMemoryBuffers.Dequeue() {
|
||||
|
||||
@ -45,7 +45,7 @@ func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Par
|
||||
b.accessLock.Lock()
|
||||
defer b.accessLock.Unlock()
|
||||
p := topic.FromPbPartition(partition)
|
||||
if localPartition:=b.localTopicManager.GetLocalPartition(t, p); localPartition!=nil {
|
||||
if localPartition := b.localTopicManager.GetLocalPartition(t, p); localPartition != nil {
|
||||
localPartition.NotifyLogFlushed(logBuffer.LastFlushTsNs)
|
||||
}
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ var (
|
||||
concurrency = flag.Int("c", 4, "concurrent publishers")
|
||||
partitionCount = flag.Int("p", 6, "partition count")
|
||||
|
||||
clientName = flag.String("client", "c1", "client name")
|
||||
clientName = flag.String("client", "c1", "client name")
|
||||
|
||||
namespace = flag.String("ns", "test", "namespace")
|
||||
t = flag.String("t", "test", "t")
|
||||
|
||||
@ -19,7 +19,7 @@ var (
|
||||
concurrency = flag.Int("c", 4, "concurrent publishers")
|
||||
partitionCount = flag.Int("p", 6, "partition count")
|
||||
|
||||
clientName = flag.String("client", "c1", "client name")
|
||||
clientName = flag.String("client", "c1", "client name")
|
||||
|
||||
namespace = flag.String("ns", "test", "namespace")
|
||||
t = flag.String("t", "test", "t")
|
||||
|
||||
@ -48,7 +48,7 @@ func (p *TopicPublisher) FinishPublish() error {
|
||||
if inputBuffers, found := p.partition2Buffer.AllIntersections(0, pub_balancer.MaxPartitionCount); found {
|
||||
for _, inputBuffer := range inputBuffers {
|
||||
inputBuffer.Enqueue(&mq_pb.DataMessage{
|
||||
TsNs: time.Now().UnixNano(),
|
||||
TsNs: time.Now().UnixNano(),
|
||||
Ctrl: &mq_pb.ControlMessage{
|
||||
IsClose: true,
|
||||
},
|
||||
|
||||
@ -17,7 +17,7 @@ type BrokerStats struct {
|
||||
}
|
||||
type TopicPartitionStats struct {
|
||||
topic.TopicPartition
|
||||
PublisherCount int32
|
||||
PublisherCount int32
|
||||
SubscriberCount int32
|
||||
}
|
||||
|
||||
@ -48,7 +48,7 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
|
||||
UnixTimeNs: topicPartitionStats.Partition.UnixTimeNs,
|
||||
},
|
||||
},
|
||||
PublisherCount: topicPartitionStats.PublisherCount,
|
||||
PublisherCount: topicPartitionStats.PublisherCount,
|
||||
SubscriberCount: topicPartitionStats.SubscriberCount,
|
||||
}
|
||||
publisherCount += topicPartitionStats.PublisherCount
|
||||
@ -76,7 +76,7 @@ func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Parti
|
||||
UnixTimeNs: partition.UnixTimeNs,
|
||||
},
|
||||
},
|
||||
PublisherCount: 0,
|
||||
PublisherCount: 0,
|
||||
SubscriberCount: 0,
|
||||
}
|
||||
key := tps.TopicPartition.String()
|
||||
|
||||
@ -6,17 +6,17 @@ import (
|
||||
|
||||
type Schema struct {
|
||||
RecordType *schema_pb.RecordType
|
||||
fieldMap map[string]*schema_pb.Field
|
||||
fieldMap map[string]*schema_pb.Field
|
||||
}
|
||||
|
||||
func NewSchema(recordType *schema_pb.RecordType) (*Schema, error) {
|
||||
fieldMap := make( map[string]*schema_pb.Field)
|
||||
fieldMap := make(map[string]*schema_pb.Field)
|
||||
for _, field := range recordType.Fields {
|
||||
fieldMap[field.Name] = field
|
||||
}
|
||||
return &Schema{
|
||||
RecordType: recordType,
|
||||
fieldMap: fieldMap,
|
||||
fieldMap: fieldMap,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@ -8,9 +8,9 @@ import (
|
||||
var (
|
||||
TypeBoolean = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_BOOL}}
|
||||
TypeInt32 = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT32}}
|
||||
TypeInt64 = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT64}}
|
||||
TypeFloat = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_FLOAT}}
|
||||
TypeDouble = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_DOUBLE}}
|
||||
TypeInt64 = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT64}}
|
||||
TypeFloat = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_FLOAT}}
|
||||
TypeDouble = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_DOUBLE}}
|
||||
TypeBytes = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_BYTES}}
|
||||
TypeString = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_STRING}}
|
||||
)
|
||||
|
||||
@ -32,10 +32,10 @@ func TestEnumScalarType(t *testing.T) {
|
||||
|
||||
func TestField(t *testing.T) {
|
||||
field := &Field{
|
||||
Name: "field_name",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
|
||||
FieldIndex: 1,
|
||||
IsRepeated: false,
|
||||
Name: "field_name",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
|
||||
FieldIndex: 1,
|
||||
IsRepeated: false,
|
||||
}
|
||||
assert.NotNil(t, field)
|
||||
}
|
||||
@ -44,32 +44,32 @@ func TestRecordType(t *testing.T) {
|
||||
subRecord := &RecordType{
|
||||
Fields: []*Field{
|
||||
{
|
||||
Name: "field_1",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
|
||||
FieldIndex: 1,
|
||||
IsRepeated: false,
|
||||
Name: "field_1",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
|
||||
FieldIndex: 1,
|
||||
IsRepeated: false,
|
||||
},
|
||||
{
|
||||
Name: "field_2",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_STRING}},
|
||||
FieldIndex: 2,
|
||||
IsRepeated: false,
|
||||
Name: "field_2",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_STRING}},
|
||||
FieldIndex: 2,
|
||||
IsRepeated: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
record := &RecordType{
|
||||
Fields: []*Field{
|
||||
{
|
||||
Name: "field_key",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
|
||||
FieldIndex: 1,
|
||||
IsRepeated: false,
|
||||
Name: "field_key",
|
||||
Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
|
||||
FieldIndex: 1,
|
||||
IsRepeated: false,
|
||||
},
|
||||
{
|
||||
Name: "field_record",
|
||||
Type: &Type{Kind: &Type_RecordType{RecordType: subRecord}},
|
||||
FieldIndex: 2,
|
||||
IsRepeated: false,
|
||||
Name: "field_record",
|
||||
Type: &Type{Kind: &Type_RecordType{RecordType: subRecord}},
|
||||
FieldIndex: 2,
|
||||
IsRepeated: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@ -76,7 +76,7 @@ func TestStructToSchema(t *testing.T) {
|
||||
RecordTypeBegin().
|
||||
WithField("Field3", TypeString).
|
||||
WithField("Field4", TypeInt32).
|
||||
RecordTypeEnd(),
|
||||
RecordTypeEnd(),
|
||||
).
|
||||
RecordTypeEnd(),
|
||||
},
|
||||
@ -104,7 +104,7 @@ func TestStructToSchema(t *testing.T) {
|
||||
RecordTypeBegin().
|
||||
WithField("Field6", TypeString).
|
||||
WithField("Field7", TypeBytes).
|
||||
RecordTypeEnd(),
|
||||
RecordTypeEnd(),
|
||||
).RecordTypeEnd(),
|
||||
).
|
||||
RecordTypeEnd(),
|
||||
|
||||
@ -7,9 +7,9 @@ import (
|
||||
|
||||
type ParquetLevels struct {
|
||||
startColumnIndex int
|
||||
endColumnIndex int
|
||||
definitionDepth int
|
||||
levels map[string]*ParquetLevels
|
||||
endColumnIndex int
|
||||
definitionDepth int
|
||||
levels map[string]*ParquetLevels
|
||||
}
|
||||
|
||||
func ToParquetLevels(recordType *schema_pb.RecordType) (*ParquetLevels, error) {
|
||||
@ -19,7 +19,7 @@ func ToParquetLevels(recordType *schema_pb.RecordType) (*ParquetLevels, error) {
|
||||
func toFieldTypeLevels(fieldType *schema_pb.Type, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
|
||||
switch fieldType.Kind.(type) {
|
||||
case *schema_pb.Type_ScalarType:
|
||||
return toFieldTypeScalarLevels(fieldType.GetScalarType(), startColumnIndex, definitionDepth)
|
||||
return toFieldTypeScalarLevels(fieldType.GetScalarType(), startColumnIndex, definitionDepth)
|
||||
case *schema_pb.Type_RecordType:
|
||||
return toRecordTypeLevels(fieldType.GetRecordType(), startColumnIndex, definitionDepth)
|
||||
case *schema_pb.Type_ListType:
|
||||
@ -35,15 +35,15 @@ func toFieldTypeListLevels(listType *schema_pb.ListType, startColumnIndex, defin
|
||||
func toFieldTypeScalarLevels(scalarType schema_pb.ScalarType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
|
||||
return &ParquetLevels{
|
||||
startColumnIndex: startColumnIndex,
|
||||
endColumnIndex: startColumnIndex + 1,
|
||||
definitionDepth: definitionDepth,
|
||||
endColumnIndex: startColumnIndex + 1,
|
||||
definitionDepth: definitionDepth,
|
||||
}, nil
|
||||
}
|
||||
func toRecordTypeLevels(recordType *schema_pb.RecordType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
|
||||
recordTypeLevels := &ParquetLevels{
|
||||
startColumnIndex: startColumnIndex,
|
||||
definitionDepth: definitionDepth,
|
||||
levels: make(map[string]*ParquetLevels),
|
||||
definitionDepth: definitionDepth,
|
||||
levels: make(map[string]*ParquetLevels),
|
||||
}
|
||||
for _, field := range recordType.Fields {
|
||||
fieldTypeLevels, err := toFieldTypeLevels(field.Type, startColumnIndex, definitionDepth+1)
|
||||
|
||||
@ -11,9 +11,9 @@ func TestToParquetLevels(t *testing.T) {
|
||||
recordType *schema_pb.RecordType
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want *ParquetLevels
|
||||
name string
|
||||
args args
|
||||
want *ParquetLevels
|
||||
}{
|
||||
{
|
||||
name: "nested type",
|
||||
@ -25,13 +25,13 @@ func TestToParquetLevels(t *testing.T) {
|
||||
RecordTypeBegin().
|
||||
WithField("zName", TypeString).
|
||||
WithField("emails", ListOf(TypeString)).
|
||||
RecordTypeEnd()).
|
||||
RecordTypeEnd()).
|
||||
WithField("Company", TypeString).
|
||||
WithRecordField("Address",
|
||||
RecordTypeBegin().
|
||||
WithField("Street", TypeString).
|
||||
WithField("City", TypeString).
|
||||
RecordTypeEnd()).
|
||||
RecordTypeEnd()).
|
||||
RecordTypeEnd(),
|
||||
},
|
||||
want: &ParquetLevels{
|
||||
|
||||
@ -31,7 +31,6 @@ func toParquetFieldType(fieldType *schema_pb.Type) (dataType parquet.Node, err e
|
||||
return nil, fmt.Errorf("unknown field type: %T", fieldType.Kind)
|
||||
}
|
||||
|
||||
|
||||
return dataType, err
|
||||
}
|
||||
|
||||
|
||||
@ -70,7 +70,7 @@ func doVisitValue(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *
|
||||
return
|
||||
}
|
||||
|
||||
func toParquetValue(value *schema_pb.Value) (parquet.Value, error) {
|
||||
func toParquetValue(value *schema_pb.Value) (parquet.Value, error) {
|
||||
switch value.Kind.(type) {
|
||||
case *schema_pb.Value_BoolValue:
|
||||
return parquet.BooleanValue(value.GetBoolValue()), nil
|
||||
|
||||
@ -47,7 +47,7 @@ func toRecordValue(recordType *schema_pb.RecordType, levels *ParquetLevels, valu
|
||||
func toListValue(listType *schema_pb.ListType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (listValue *schema_pb.Value, endValueIndex int, err error) {
|
||||
listValues := make([]*schema_pb.Value, 0)
|
||||
var value *schema_pb.Value
|
||||
for ;valueIndex < len(values); {
|
||||
for valueIndex < len(values) {
|
||||
if values[valueIndex].Column() != levels.startColumnIndex {
|
||||
break
|
||||
}
|
||||
@ -67,19 +67,19 @@ func toScalarValue(scalarType schema_pb.ScalarType, levels *ParquetLevels, value
|
||||
}
|
||||
switch scalarType {
|
||||
case schema_pb.ScalarType_BOOL:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value.Boolean()}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value.Boolean()}}, valueIndex + 1, nil
|
||||
case schema_pb.ScalarType_INT32:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value.Int32()}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value.Int32()}}, valueIndex + 1, nil
|
||||
case schema_pb.ScalarType_INT64:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value.Int64()}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value.Int64()}}, valueIndex + 1, nil
|
||||
case schema_pb.ScalarType_FLOAT:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value.Float()}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value.Float()}}, valueIndex + 1, nil
|
||||
case schema_pb.ScalarType_DOUBLE:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value.Double()}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value.Double()}}, valueIndex + 1, nil
|
||||
case schema_pb.ScalarType_BYTES:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value.ByteArray()}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value.ByteArray()}}, valueIndex + 1, nil
|
||||
case schema_pb.ScalarType_STRING:
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: string(value.ByteArray())}}, valueIndex+1, nil
|
||||
return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: string(value.ByteArray())}}, valueIndex + 1, nil
|
||||
}
|
||||
return nil, valueIndex, fmt.Errorf("unsupported scalar type: %v", scalarType)
|
||||
}
|
||||
|
||||
@ -19,13 +19,13 @@ func TestWriteReadParquet(t *testing.T) {
|
||||
RecordTypeBegin().
|
||||
WithField("zName", TypeString).
|
||||
WithField("emails", ListOf(TypeString)).
|
||||
RecordTypeEnd()).
|
||||
RecordTypeEnd()).
|
||||
WithField("Company", TypeString).
|
||||
WithRecordField("Address",
|
||||
RecordTypeBegin().
|
||||
WithField("Street", TypeString).
|
||||
WithField("City", TypeString).
|
||||
RecordTypeEnd()).
|
||||
RecordTypeEnd()).
|
||||
RecordTypeEnd()
|
||||
fmt.Printf("RecordType: %v\n", recordType)
|
||||
|
||||
@ -85,9 +85,9 @@ func testWritingParquetFile(t *testing.T, count int, filename string, parquetSch
|
||||
fmt.Sprintf("john_%d@c.com", i),
|
||||
fmt.Sprintf("john_%d@d.com", i),
|
||||
fmt.Sprintf("john_%d@e.com", i)).
|
||||
RecordEnd()).
|
||||
RecordEnd()).
|
||||
SetString("Company", fmt.Sprintf("company_%d", i)).
|
||||
RecordEnd()
|
||||
RecordEnd()
|
||||
AddRecordValue(rowBuilder, recordType, parquetLevels, recordValue)
|
||||
|
||||
if count < 10 {
|
||||
|
||||
@ -16,17 +16,17 @@ import (
|
||||
)
|
||||
|
||||
type LocalPartition struct {
|
||||
ListenersWaits int64
|
||||
AckTsNs int64
|
||||
ListenersWaits int64
|
||||
AckTsNs int64
|
||||
|
||||
// notifying clients
|
||||
ListenersLock sync.Mutex
|
||||
ListenersCond *sync.Cond
|
||||
|
||||
Partition
|
||||
LogBuffer *log_buffer.LogBuffer
|
||||
Publishers *LocalPartitionPublishers
|
||||
Subscribers *LocalPartitionSubscribers
|
||||
LogBuffer *log_buffer.LogBuffer
|
||||
Publishers *LocalPartitionPublishers
|
||||
Subscribers *LocalPartitionSubscribers
|
||||
|
||||
publishFolloweMeStream mq_pb.SeaweedMessaging_PublishFollowMeClient
|
||||
followerGrpcConnection *grpc.ClientConn
|
||||
@ -37,7 +37,7 @@ var TIME_FORMAT = "2006-01-02-15-04-05"
|
||||
|
||||
func NewLocalPartition(partition Partition, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
|
||||
lp := &LocalPartition{
|
||||
Partition: partition,
|
||||
Partition: partition,
|
||||
Publishers: NewLocalPartitionPublishers(),
|
||||
Subscribers: NewLocalPartitionSubscribers(),
|
||||
}
|
||||
@ -155,8 +155,8 @@ func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessa
|
||||
if err = p.publishFolloweMeStream.Send(&mq_pb.PublishFollowMeRequest{
|
||||
Message: &mq_pb.PublishFollowMeRequest_Init{
|
||||
Init: &mq_pb.PublishFollowMeRequest_InitMessage{
|
||||
Topic: initMessage.Topic,
|
||||
Partition: initMessage.Partition,
|
||||
Topic: initMessage.Topic,
|
||||
Partition: initMessage.Partition,
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"strconv"
|
||||
"strings"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@ -202,7 +203,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
|
||||
Tagging: aws.String(tags),
|
||||
}
|
||||
if len(entry.Attributes.Md5) > 0 {
|
||||
uploadInput.ContentMD5 = aws.String(fmt.Sprintf("%x", entry.Attributes.Md5))
|
||||
uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
|
||||
}
|
||||
_, err = uploader.Upload(&uploadInput)
|
||||
|
||||
|
||||
@ -317,6 +317,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
|
||||
}
|
||||
|
||||
identity, errCode := iam.authRequest(r, action)
|
||||
glog.V(3).Infof("auth error: %v", errCode)
|
||||
if errCode == s3err.ErrNone {
|
||||
if identity != nil && identity.Name != "" {
|
||||
r.Header.Set(s3_constants.AmzIdentityId, identity.Name)
|
||||
@ -453,6 +454,7 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string)
|
||||
}
|
||||
}
|
||||
if bucket == "" {
|
||||
glog.V(3).Infof("identity %s is not allowed to perform action %s on %s -- bucket is empty", identity.Name, action, bucket+objectKey)
|
||||
return false
|
||||
}
|
||||
target := string(action) + ":" + bucket + objectKey
|
||||
@ -477,6 +479,8 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string)
|
||||
}
|
||||
}
|
||||
}
|
||||
//log error
|
||||
glog.V(3).Infof("identity %s is not allowed to perform action %s on %s", identity.Name, action, bucket+objectKey)
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
jsonpb "google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
@ -79,6 +80,7 @@ func TestCanDo(t *testing.T) {
|
||||
}
|
||||
// object specific
|
||||
assert.Equal(t, true, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/c/d.txt"))
|
||||
assert.Equal(t, false, ident1.canDo(ACTION_DELETE_BUCKET, "bucket1", ""))
|
||||
assert.Equal(t, false, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/other/some"), "action without *")
|
||||
|
||||
// bucket specific
|
||||
@ -141,6 +143,15 @@ func TestCanDo(t *testing.T) {
|
||||
},
|
||||
}
|
||||
assert.Equal(t, true, ident6.canDo(ACTION_READ, "anything_bucket", "/a/b/c/d.txt"))
|
||||
|
||||
//test deleteBucket operation
|
||||
ident7 := &Identity{
|
||||
Name: "anything",
|
||||
Actions: []Action{
|
||||
"DeleteBucket:bucket1",
|
||||
},
|
||||
}
|
||||
assert.Equal(t, true, ident7.canDo(ACTION_DELETE_BUCKET, "bucket1", ""))
|
||||
}
|
||||
|
||||
type LoadS3ApiConfigurationTestCase struct {
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
package s3_constants
|
||||
|
||||
const (
|
||||
ACTION_READ = "Read"
|
||||
ACTION_READ_ACP = "ReadAcp"
|
||||
ACTION_WRITE = "Write"
|
||||
ACTION_WRITE_ACP = "WriteAcp"
|
||||
ACTION_ADMIN = "Admin"
|
||||
ACTION_TAGGING = "Tagging"
|
||||
ACTION_LIST = "List"
|
||||
ACTION_READ = "Read"
|
||||
ACTION_READ_ACP = "ReadAcp"
|
||||
ACTION_WRITE = "Write"
|
||||
ACTION_WRITE_ACP = "WriteAcp"
|
||||
ACTION_ADMIN = "Admin"
|
||||
ACTION_TAGGING = "Tagging"
|
||||
ACTION_LIST = "List"
|
||||
ACTION_DELETE_BUCKET = "DeleteBucket"
|
||||
|
||||
SeaweedStorageDestinationHeader = "x-seaweedfs-destination"
|
||||
MultipartUploadsFolder = ".uploads"
|
||||
|
||||
@ -7,7 +7,7 @@ import (
|
||||
var (
|
||||
CircuitBreakerConfigDir = "/etc/s3"
|
||||
CircuitBreakerConfigFile = "circuit_breaker.json"
|
||||
AllowedActions = []string{ACTION_READ, ACTION_READ_ACP, ACTION_WRITE, ACTION_WRITE_ACP, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN}
|
||||
AllowedActions = []string{ACTION_READ, ACTION_READ_ACP, ACTION_WRITE, ACTION_WRITE_ACP, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN, ACTION_DELETE_BUCKET}
|
||||
LimitTypeCount = "Count"
|
||||
LimitTypeBytes = "MB"
|
||||
Separator = ":"
|
||||
|
||||
@ -6,14 +6,15 @@ import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
@ -218,6 +219,10 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC
|
||||
return s3err.ErrNoSuchBucket
|
||||
}
|
||||
|
||||
//if iam is enabled, the access was already checked before
|
||||
if s3a.iam.isEnabled() {
|
||||
return s3err.ErrNone
|
||||
}
|
||||
if !s3a.hasAccess(r, entry) {
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
@ -236,6 +241,7 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
|
||||
identityId := r.Header.Get(s3_constants.AmzIdentityId)
|
||||
if id, ok := entry.Extended[s3_constants.AmzIdentityId]; ok {
|
||||
if identityId != string(id) {
|
||||
glog.V(3).Infof("hasAccess: %s != %s (entry.Extended = %v)", identityId, id, entry.Extended)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@ -279,7 +279,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
|
||||
bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT"))
|
||||
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_ADMIN)), "DELETE"))
|
||||
bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE"))
|
||||
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST"))
|
||||
|
||||
@ -278,7 +278,7 @@ func TestDeleteEmptySelection(t *testing.T) {
|
||||
eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
|
||||
for _, diskInfo := range dn.DiskInfos {
|
||||
for _, v := range diskInfo.VolumeInfos {
|
||||
if v.Size <= super_block.SuperBlockSize && v.ModifiedAtSecond > 0 {
|
||||
if v.Size <= super_block.SuperBlockSize && v.ModifiedAtSecond > 0 {
|
||||
fmt.Printf("empty volume %d from %s\n", v.Id, dn.Id)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
//go:build openbsd || netbsd || plan9 || solaris
|
||||
// +build openbsd netbsd plan9 solaris
|
||||
//go:build netbsd || plan9 || solaris
|
||||
// +build netbsd plan9 solaris
|
||||
|
||||
package stats
|
||||
|
||||
|
||||
25
weed/stats/disk_openbsd.go
Normal file
25
weed/stats/disk_openbsd.go
Normal file
@ -0,0 +1,25 @@
|
||||
//go:build openbsd
|
||||
// +build openbsd
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
)
|
||||
|
||||
func fillInDiskStatus(disk *volume_server_pb.DiskStatus) {
|
||||
fs := syscall.Statfs_t{}
|
||||
err := syscall.Statfs(disk.Dir, &fs)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
disk.All = fs.F_blocks * uint64(fs.F_bsize)
|
||||
disk.Free = fs.F_bfree * uint64(fs.F_bsize)
|
||||
disk.Used = disk.All - disk.Free
|
||||
disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
|
||||
disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
|
||||
return
|
||||
}
|
||||
|
||||
@ -36,8 +36,8 @@ func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
|
||||
func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
|
||||
e := idx.WalkIndexFile(file, 0, func(key NeedleId, offset Offset, size Size) error {
|
||||
nm.MaybeSetMaxFileKey(key)
|
||||
nm.FileCounter++
|
||||
if !offset.IsZero() && size.IsValid() {
|
||||
nm.FileCounter++
|
||||
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
|
||||
oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
|
||||
if !oldOffset.IsZero() && oldSize.IsValid() {
|
||||
@ -51,7 +51,7 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
glog.V(1).Infof("max file key: %d for file: %s", nm.MaxFileKey(), file.Name())
|
||||
glog.V(1).Infof("max file key: %v count: %d deleted: %d for file: %s", nm.MaxFileKey(), nm.FileCount(), nm.DeletedCount(), file.Name())
|
||||
return nm, e
|
||||
}
|
||||
|
||||
|
||||
@ -109,9 +109,6 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version,
|
||||
return 0, fmt.Errorf("verifyNeedleIntegrity check %s entry offset %d size %d: %v", datFile.Name(), offset, size, err)
|
||||
}
|
||||
n.AppendAtNs = util.BytesToUint64(bytes)
|
||||
if n.HasTtl() {
|
||||
return n.AppendAtNs, nil
|
||||
}
|
||||
fileTailOffset := offset + needle.GetActualSize(size, v)
|
||||
fileSize, _, err := datFile.GetStat()
|
||||
if err != nil {
|
||||
@ -130,7 +127,7 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version,
|
||||
return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err)
|
||||
}
|
||||
if n.Id != key {
|
||||
return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
|
||||
return n.AppendAtNs, fmt.Errorf("index key %v does not match needle's Id %v", key, n.Id)
|
||||
}
|
||||
return n.AppendAtNs, err
|
||||
}
|
||||
@ -147,7 +144,7 @@ func verifyDeletedNeedleIntegrity(datFile backend.BackendStorageFile, v needle.V
|
||||
return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", fileSize-size, size, err)
|
||||
}
|
||||
if n.Id != key {
|
||||
return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
|
||||
return n.AppendAtNs, fmt.Errorf("index key %v does not match needle's Id %v", key, n.Id)
|
||||
}
|
||||
return n.AppendAtNs, err
|
||||
}
|
||||
|
||||
@ -487,19 +487,21 @@ func (v *Volume) copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, da
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dstDatSize, _, err := dstDatBackend.GetStat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v.nm.ContentSize() > v.nm.DeletedSize() {
|
||||
expectedContentSize := v.nm.ContentSize() - v.nm.DeletedSize()
|
||||
if expectedContentSize > uint64(dstDatSize) {
|
||||
return fmt.Errorf("volume %s unexpected new data size: %d does not match size of content minus deleted: %d",
|
||||
v.Id.String(), dstDatSize, expectedContentSize)
|
||||
if v.Ttl.String() == "" {
|
||||
dstDatSize, _, err := dstDatBackend.GetStat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v.nm.ContentSize() > v.nm.DeletedSize() {
|
||||
expectedContentSize := v.nm.ContentSize() - v.nm.DeletedSize()
|
||||
if expectedContentSize > uint64(dstDatSize) {
|
||||
return fmt.Errorf("volume %s unexpected new data size: %d does not match size of content minus deleted: %d",
|
||||
v.Id.String(), dstDatSize, expectedContentSize)
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("volume %s content size: %d less deleted size: %d, new size: %d",
|
||||
v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize)
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("volume %s content size: %d less deleted size: %d, new size: %d",
|
||||
v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize)
|
||||
}
|
||||
err = newNm.SaveToIdx(datIdxName)
|
||||
if err != nil {
|
||||
|
||||
@ -31,20 +31,20 @@ type VolumeGrowRequest struct {
|
||||
}
|
||||
|
||||
type volumeGrowthStrategy struct {
|
||||
Copy1Count int
|
||||
Copy2Count int
|
||||
Copy3Count int
|
||||
Copy1Count int
|
||||
Copy2Count int
|
||||
Copy3Count int
|
||||
CopyOtherCount int
|
||||
Threshold float64
|
||||
Threshold float64
|
||||
}
|
||||
|
||||
var (
|
||||
VolumeGrowStrategy = volumeGrowthStrategy{
|
||||
Copy1Count: 7,
|
||||
Copy2Count: 6,
|
||||
Copy3Count: 3,
|
||||
Copy1Count: 7,
|
||||
Copy2Count: 6,
|
||||
Copy3Count: 3,
|
||||
CopyOtherCount: 1,
|
||||
Threshold: 0.9,
|
||||
Threshold: 0.9,
|
||||
}
|
||||
)
|
||||
|
||||
@ -77,7 +77,8 @@ func NewDefaultVolumeGrowth() *VolumeGrowth {
|
||||
// given copyCount, how many logical volumes to create
|
||||
func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) {
|
||||
switch copyCount {
|
||||
case 1: count = VolumeGrowStrategy.Copy1Count
|
||||
case 1:
|
||||
count = VolumeGrowStrategy.Copy1Count
|
||||
case 2:
|
||||
count = VolumeGrowStrategy.Copy2Count
|
||||
case 3:
|
||||
|
||||
@ -381,7 +381,7 @@ func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (total, a
|
||||
}
|
||||
active++
|
||||
info, _ := dn.GetVolumesById(v)
|
||||
if float64(info.Size) > float64(vl.volumeSizeLimit)* VolumeGrowStrategy.Threshold{
|
||||
if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold {
|
||||
crowded++
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,7 +19,7 @@ func TestNewLogBufferFirstBuffer(t *testing.T) {
|
||||
}, nil, func() {
|
||||
})
|
||||
|
||||
startTime := MessagePosition{Time:time.Now()}
|
||||
startTime := MessagePosition{Time: time.Now()}
|
||||
|
||||
messageSize := 1024
|
||||
messageCount := 5000
|
||||
@ -38,7 +38,7 @@ func TestNewLogBufferFirstBuffer(t *testing.T) {
|
||||
println("processed all messages")
|
||||
return true, io.EOF
|
||||
}
|
||||
return false,nil
|
||||
return false, nil
|
||||
})
|
||||
|
||||
fmt.Printf("before flush: sent %d received %d\n", messageCount, receivedMessageCount)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user