Enable multiple disks per volume server. Also streamline the use of /logs & /idx (#5236)

* Enable multiple disks per volume server. Also streamline the use of /logs and /idx * missing file * add loop around volume mounts for initContainers * remove uneeded name for idx
2025-04-05 20:52:50 +08:00 · 2024-01-24 18:25:17 +01:00 · 2024-01-24 18:25:17 +01:00 · 95787ea89d
commit 95787ea89d
parent 025f8eb37b
3 changed files with 88 additions and 70 deletions
--- a/k8s/charts/seaweedfs/templates/_helpers.tpl
+++ b/k8s/charts/seaweedfs/templates/_helpers.tpl
@ -142,7 +142,7 @@ Inject extra environment vars in the format key:value, if populated

 {{/* check if any InitContainers exist for Volumes */}}
 {{- define "volume.initContainers_exists" -}}
-{{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}}
+{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
 {{- printf "true" -}}
 {{- else -}}
 {{- printf "" -}}
--- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml
@ -54,17 +54,19 @@ spec:
      {{- $initContainers_exists := include "volume.initContainers_exists" . -}}
      {{- if $initContainers_exists }}
      initContainers:
-        {{- if .Values.volume.dir_idx }}
+        {{- if .Values.volume.idx }}
        - name: seaweedfs-vol-move-idx
          image: {{ template "volume.image" . }}
          imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }}
          command: [ '/bin/sh', '-c' ]
-          args: [ 'if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;' ]
+          args: [ 'ls {{range $dir :=  .Values.volume.dataDirs }}/{{$dir.name}}/*.idx {{end}} && xargs -I {} mv {} /idx/' ]
          volumeMounts:
            - name: idx
-              mountPath: {{ .Values.volume.dir_idx }}
-            - name: data
-              mountPath: {{ .Values.volume.dir }}
+              mountPath: {{ .Values.volume.idx.name }}
+          {{- range $dir :=  .Values.volume.dataDirs }} 
+            - name: {{ $dir.name }}
+              mountPath: /{{ $dir.name }}
+          {{- end }}
        {{- end }}
        {{- if .Values.volume.initContainers }}
        {{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
@ -100,7 +102,7 @@ spec:
            - "-ec"
            - |
              exec /usr/bin/weed \
-                {{- if eq .Values.volume.logs.type "hostPath" }}
+                {{- if .Values.volume.logs }}
                -logdir=/logs \
                {{- else }}
                -logtostderr=true \
@ -115,11 +117,11 @@ spec:
                {{- if .Values.volume.metricsPort }}
                -metricsPort={{ .Values.volume.metricsPort }} \
                {{- end }}
-                -dir={{ .Values.volume.dir }} \
-                {{- if .Values.volume.dir_idx }}
-                -dir.idx={{ .Values.volume.dir_idx }} \
+                -dir {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \
+                {{- if .Values.volume.idx }}
+                -dir.idx= /idx \
                {{- end }}
-                -max={{ .Values.volume.maxVolumes }} \
+                -max {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}{{$dir.maxVolumes}}{{end}} \
                {{- if .Values.volume.rack }}
                -rack={{ .Values.volume.rack }} \
                {{- end }}
@ -148,16 +150,18 @@ spec:
                -compactionMBps={{ .Values.volume.compactionMBps }} \
                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
          volumeMounts:
-            - name: data
-              mountPath: "{{ .Values.volume.dir }}/"
-            {{- if .Values.volume.dir_idx }}
-            - name: idx
-              mountPath: "{{ .Values.volume.dir_idx }}/"
+            {{- range $dir := .Values.volume.dataDirs }}
+            - name: {{ $dir.name }}
+              mountPath: "/{{ $dir.name }}/"
            {{- end }}
-            {{- if eq .Values.volume.logs.type "hostPath" }}
+            {{- if .Values.volume.logs }}
            - name: logs
              mountPath: "/logs/"
            {{- end }}
+            {{- if .Values.volume.idx }}
+            - name: idx
+              mountPath: "/idx/"
+            {{- end }}
            {{- if .Values.global.enableSecurity }}
            - name: security-config
              readOnly: true
@ -221,40 +225,51 @@ spec:
      {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
-        {{- if eq .Values.volume.data.type "hostPath" }}
-        - name: data
+
+     {{- range $dir := .Values.volume.dataDirs }}
+
+      {{- if eq $dir.type "hostPath" }}
+        - name: {{ $dir.name }}
          hostPath:
-            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
+            path: {{ $dir.hostPathPrefix }}/object_store/
            type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.data.type "existingClaim" }}
-        - name: data
+      {{- end }}
+      {{- if eq $dir.type "existingClaim" }}
+        - name: {{ $dir.name }}
          persistentVolumeClaim:
-            claimName: {{ .Values.volume.data.claimName }} 
-        {{- end }}
-        {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
+            claimName: {{ $dir.claimName }} 
+      {{- end }}
+
+     {{- end }}
+
+     {{- if .Values.volume.idx }}
+       {{- if eq .Values.volume.idx.type "hostPath" }}
        - name: idx
          hostPath:
            path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
            type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.idx.type "existingClaim" }}
+       {{- end }}
+       {{- if eq .Values.volume.idx.type "existingClaim" }}
        - name: idx
          persistentVolumeClaim:
            claimName: {{ .Values.volume.idx.claimName }} 
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "hostPath" }}
+       {{- end }}
+     {{- end }}
+
+     {{- if .Values.volume.logs }}
+       {{- if eq .Values.volume.logs.type "hostPath" }}
        - name: logs
          hostPath:
            path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
            type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "existingClaim" }}
+       {{- end }}
+       {{- if eq .Values.volume.logs.type "existingClaim" }}
        - name: logs
          persistentVolumeClaim:
-            claimName: {{ .Values.volume.data.claimName }} 
-        {{- end }}
-        {{- if .Values.global.enableSecurity }}
+            claimName: {{ .Values.volume.logs.claimName }} 
+       {{- end }}
+     {{- end }}
+     {{- if .Values.global.enableSecurity }}
        - name: security-config
          configMap:
            name: {{ template "seaweedfs.name" . }}-security-config
@ -273,7 +288,7 @@ spec:
        - name: client-cert
          secret:
            secretName: {{ template "seaweedfs.name" . }}-client-cert
-        {{- end }}
+      {{- end }}
      {{- if .Values.volume.extraVolumes }}
        {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
      {{- end }}
@ -281,24 +296,25 @@ spec:
      nodeSelector:
        {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
      {{- end }}
-  {{- $pvc_exists := include "volume.pvc_exists" . -}}
-  {{- if $pvc_exists }}
  volumeClaimTemplates:
-    {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
+    {{- range $dir := .Values.volume.dataDirs }}
+    {{- if eq $dir.type "persistentVolumeClaim" }}
    - metadata:
-        name: data
-        {{- with .Values.volume.data.annotations }}
+        name: {{ $dir.name }}
+        {{- with $dir.annotations }}
        annotations:
          {{- toYaml . | nindent 10 }}
        {{- end }}
      spec:
        accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.data.storageClass }}
+        storageClassName: {{ $dir.storageClass }}
        resources:
          requests:
-            storage: {{ .Values.volume.data.size }}
+            storage: {{ $dir.size }}
    {{- end }}
-    {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
+    {{- end }}
+
+    {{- if and .Values.volume.idx (eq .Values.volume.idx.type "persistentVolumeClaim") }}
    - metadata:
        name: idx
        {{- with .Values.volume.idx.annotations }}
@ -312,7 +328,7 @@ spec:
          requests:
            storage: {{ .Values.volume.idx.size }}
    {{- end }}
-    {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
+    {{- if and .Values.volume.logs (eq .Values.volume.logs.type "persistentVolumeClaim") }}
    - metadata:
        name: logs
        {{- with .Values.volume.logs.annotations }}
@ -326,5 +342,4 @@ spec:
          requests:
            storage: {{ .Values.volume.logs.size }}
    {{- end }}
-      {{- end }}
-{{- end }}
+    {{- end }}
--- a/k8s/charts/seaweedfs/values.yaml
+++ b/k8s/charts/seaweedfs/values.yaml
@ -89,7 +89,6 @@ master:
  #    claimName: "my-pvc"
  data:
    type: "hostPath"
-    size: ""
    storageClass: ""
    hostPathPrefix: /ssd

@ -239,25 +238,36 @@ volume:
  # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
  minFreeSpacePercent: 7

-  # You may use ANY storage-class, example with local-path-provisioner
+  # For each data disk you may use ANY storage-class, example with local-path-provisioner
  # Annotations are optional.
-  #  data:
-  #    type: "persistentVolumeClaim"
-  #    size: "24Ti"
-  #    storageClass: "local-path-provisioner"
-  #    annotations:
+  #  dataDirs:
+  #   - name: data:
+  #     type: "persistentVolumeClaim"
+  #     size: "24Ti"
+  #     storageClass: "local-path-provisioner"
+  #     annotations:
  #      "key": "value"
+  #     maxVolumes: 0
  #
  # You may also spacify an existing claim:
-  #  data:
-  #    type: "existingClaim"
-  #    claimName: "my-pvc"
+  #    - name: data
+  #      type: "existingClaim"
+  #      claimName: "my-pvc"
+  #     maxVolumes: 0
+
+
+  dataDirs:
+   - name: data1
+     type: "hostPath"
+     hostPathPrefix: /ssd
+     maxVolumes: 0   # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+
+   #- name: data2
+   #  type: "persistentVolumeClaim"
+   #  storageClass: "yourClassNameOfChoice"
+   #  size: "800Gi"
+   #  maxVolumes: 0

-  data:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-    hostPathPrefix: /storage

  idx:
    type: "hostPath"
@ -265,6 +275,7 @@ volume:
    storageClass: ""
    hostPathPrefix: /ssd

+
  logs:
    type: "hostPath"
    size: ""
@ -274,14 +285,6 @@ volume:
  # limit background compaction or copying speed in mega bytes per second
  compactionMBps: "50"

-  # Directories to store data files. dir[,dir]... (default "/tmp")
-  dir: "/data"
-  # Directories to store index files. dir[,dir]... (default is the same as "dir")
-  dir_idx: null
-
-  # Maximum numbers of volumes, count[,count]...
-  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
-  maxVolumes: "0"

  # Volume server's rack name
  rack: null