From 831794f535522a21a1139b7723f84c60b3b47c90 Mon Sep 17 00:00:00 2001 From: ColorfulDick <1123063417@qq.com> Date: Wed, 28 Sep 2022 17:47:39 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0juicefs=E4=BD=9C=E4=B8=BA?= =?UTF-8?q?=E5=88=86=E5=B8=83=E5=BC=8F=E5=85=B1=E4=BA=AB=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=9A=84=E7=BC=96=E6=8E=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- install/kubernetes/juicefs/.env | 4 + install/kubernetes/juicefs/README.md | 3 + .../infra-pv-pvc-with-juicefs.yml | 88 +++ .../jupyter-pv-pv-with-juicefs.yml | 90 +++ .../katib-pv-pvc-with-juicefs.yml | 45 ++ .../kubeflow-pv-pvc-with-juicefs.yml | 44 ++ .../pipeline-pv-pvc-wiht-juicefs.yml | 136 +++++ .../service-pv-pvc-with-juicefs.yml | 87 +++ install/kubernetes/juicefs/docker-compose.yml | 35 ++ .../juicefs-decive-of-k8s/juicefs-sc.yml | 28 + .../juicefs-decive-of-k8s/k8s_decive.yml | 524 ++++++++++++++++++ install/kubernetes/juicefs/start_juicefs.sh | 31 ++ 12 files changed, 1115 insertions(+) create mode 100644 install/kubernetes/juicefs/.env create mode 100644 install/kubernetes/juicefs/README.md create mode 100644 install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/infra-pv-pvc-with-juicefs.yml create mode 100644 install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/jupyter-pv-pv-with-juicefs.yml create mode 100644 install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/katib-pv-pvc-with-juicefs.yml create mode 100644 install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/kubeflow-pv-pvc-with-juicefs.yml create mode 100644 install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/pipeline-pv-pvc-wiht-juicefs.yml create mode 100644 install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/service-pv-pvc-with-juicefs.yml create mode 100644 install/kubernetes/juicefs/docker-compose.yml create mode 100644 install/kubernetes/juicefs/juicefs-decive-of-k8s/juicefs-sc.yml create mode 100644 install/kubernetes/juicefs/juicefs-decive-of-k8s/k8s_decive.yml create mode 100644 install/kubernetes/juicefs/start_juicefs.sh diff --git a/install/kubernetes/juicefs/.env b/install/kubernetes/juicefs/.env new file mode 100644 index 00000000..d5a8394e --- /dev/null +++ b/install/kubernetes/juicefs/.env @@ -0,0 +1,4 @@ +JUICEFS_HOST_IP=10.48.60.91 +REDIS_PASSWORD=myredispassword +MINIO_ROOT_USER=root +MINIO_ROOT_PASSWORD=Dewe_2131 \ No newline at end of file diff --git a/install/kubernetes/juicefs/README.md b/install/kubernetes/juicefs/README.md new file mode 100644 index 00000000..f7762733 --- /dev/null +++ b/install/kubernetes/juicefs/README.md @@ -0,0 +1,3 @@ +### 如何以juicefs作为cube-stuido的训练、部署pv的共享目录： +- 1，修改目录(juicefs)中的.env文件，将JUICEFS_HOST_IP改为自己部署cube-studio的节点的ip地址； +- 2，在控制台执行sh start_juicefs.sh \ No newline at end of file diff --git a/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/infra-pv-pvc-with-juicefs.yml b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/infra-pv-pvc-with-juicefs.yml new file mode 100644 index 00000000..cf3163a7 --- /dev/null +++ b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/infra-pv-pvc-with-juicefs.yml @@ -0,0 +1,88 @@ +# 平台上传需要 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: infra-kubeflow-global-pv + labels: + infra-pvname: infra-kubeflow-global-pv +spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: infra-kubeflow-global-pv + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/global +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-global-pvc + namespace: infra +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi + selector: + matchLabels: + infra-pvname: infra-kubeflow-global-pv + + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: infra-kubeflow + labels: + infra-pvname: infra-kubeflow +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: infra-kubeflow + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: infra-kubeflow + namespace: infra +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + selector: + matchLabels: + infra-pvname: infra-kubeflow + diff --git a/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/jupyter-pv-pv-with-juicefs.yml b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/jupyter-pv-pv-with-juicefs.yml new file mode 100644 index 00000000..fb8bcf38 --- /dev/null +++ b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/jupyter-pv-pv-with-juicefs.yml @@ -0,0 +1,90 @@ + +# 模型训练 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: jupyter-kubeflow-user-workspace + labels: + jupyter-pvname: jupyter-kubeflow-user-workspace +spec: + capacity: + storage: 500Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: jupyter-kubeflow-user-workspace + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/workspace +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-user-workspace + namespace: jupyter +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + jupyter-pvname: jupyter-kubeflow-user-workspace + + +# 模型归档 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: jupyter-kubeflow-archives + labels: + jupyter-pvname: jupyter-kubeflow-archives +spec: + capacity: + storage: 500Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: jupyter-kubeflow-archives + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/archives +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-archives + namespace: jupyter +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + jupyter-pvname: jupyter-kubeflow-archives + diff --git a/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/katib-pv-pvc-with-juicefs.yml b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/katib-pv-pvc-with-juicefs.yml new file mode 100644 index 00000000..c07ba87e --- /dev/null +++ b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/katib-pv-pvc-with-juicefs.yml @@ -0,0 +1,45 @@ + + +# 模型训练 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: katib-kubeflow-user-workspace + labels: + katib-pvname: katib-kubeflow-user-workspace +spec: + capacity: + storage: 500Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: katib-kubeflow-user-workspace + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/workspace +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-user-workspace + namespace: katib +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + katib-pvname: katib-kubeflow-user-workspace diff --git a/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/kubeflow-pv-pvc-with-juicefs.yml b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/kubeflow-pv-pvc-with-juicefs.yml new file mode 100644 index 00000000..90004cf4 --- /dev/null +++ b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/kubeflow-pv-pvc-with-juicefs.yml @@ -0,0 +1,44 @@ +# 模型训练 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: kubeflow-kubeflow-user-workspace + labels: + kubeflow-pvname: kubeflow-kubeflow-user-workspace +spec: +# storageClassName: pipeline + capacity: + storage: 500Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: kubeflow-kubeflow-user-workspace + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/workspace +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-user-workspace + namespace: kubeflow +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + kubeflow-pvname: kubeflow-kubeflow-user-workspace \ No newline at end of file diff --git a/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/pipeline-pv-pvc-wiht-juicefs.yml b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/pipeline-pv-pvc-wiht-juicefs.yml new file mode 100644 index 00000000..1882d15f --- /dev/null +++ b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/pipeline-pv-pvc-wiht-juicefs.yml @@ -0,0 +1,136 @@ +# 训练调用需要 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pipeline-kubeflow-global-pv + labels: + pipeline-pvname: pipeline-kubeflow-global-pv +spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: pipeline-kubeflow-global-pv + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/global +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-global-pvc + namespace: pipeline +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi + selector: + matchLabels: + pipeline-pvname: pipeline-kubeflow-global-pv + + +# 模型训练 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pipeline-kubeflow-user-workspace + labels: + pipeline-pvname: pipeline-kubeflow-user-workspace +spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: pipeline-kubeflow-user-workspace + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/workspace +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-user-workspace + namespace: pipeline +spec: + storageClassName: "" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + pipeline-pvname: pipeline-kubeflow-user-workspace + +# 模型归档 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pipeline-kubeflow-archives + labels: + pipeline-pvname: pipeline-kubeflow-archives +spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: pipeline-kubeflow-archives + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/archives +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-archives + namespace: pipeline +spec: + storageClassName: "" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + pipeline-pvname: pipeline-kubeflow-archives diff --git a/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/service-pv-pvc-with-juicefs.yml b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/service-pv-pvc-with-juicefs.yml new file mode 100644 index 00000000..ff428423 --- /dev/null +++ b/install/kubernetes/juicefs/cube-pv-pvc-with-juicefs/service-pv-pvc-with-juicefs.yml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: service-kubeflow-user-workspace + labels: + service-pvname: service-kubeflow-user-workspace +spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: service-kubeflow-user-workspace + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/workspace +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: service-kubeflow-archives + labels: + service-pvname: service-kubeflow-archives +spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: csi.juicefs.com + volumeHandle: service-kubeflow-archives + fsType: juicefs + nodePublishSecretRef: + name: juicefs-sc-secret + namespace: kube-system + volumeAttributes: + juicefs/mount-cpu-limit: 5000m + juicefs/mount-memory-limit: 5Gi + juicefs/mount-cpu-request: 1m + juicefs/mount-memory-request: 1Mi + mountOptions: + - subdir=kubeflow/pipeline/archives +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-user-workspace + namespace: service +spec: + storageClassName: "" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + service-pvname: service-kubeflow-user-workspace +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kubeflow-archives + namespace: service +spec: + storageClassName: "" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Gi + selector: + matchLabels: + service-pvname: service-kubeflow-archives \ No newline at end of file diff --git a/install/kubernetes/juicefs/docker-compose.yml b/install/kubernetes/juicefs/docker-compose.yml new file mode 100644 index 00000000..73114c15 --- /dev/null +++ b/install/kubernetes/juicefs/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3' + +services: + minio-juicefs: + container_name: minio-juicefs + restart: always + image: bitnami/minio:2022.9.17 + #ports: #使用host模式时不需要 + # - '9010:9000' + # - '9011:9001' + volumes: + - /data/allvolumes/minio/minio_data:/data + #- /data/allvolumes/minio/minio_config:/root/.minio + command: minio server --address ${JUICEFS_HOST_IP}:9010 --console-address ${JUICEFS_HOST_IP}:9011 /data + environment: + MINIO_ACCESS_KEY: ${MINIO_ROOT_USER} #用于外部接口调用 + MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD} + MINIO_ROOT_USER: ${MINIO_ROOT_USER} #用于登录管理页面 + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_DEFAULT_BUCKETS: "juicefs:policy" #默认得bucket + network_mode: host + + redis-juicefs: + image: redis:7.0.4-alpine3.16 + container_name: redis-juicefs + restart: always + #expose: + # - 6379 + ports: #使用host模式时不需要 + - 6382:6379 + restart: always + #network_mode: host + volumes: + - /data/allvolumes/redis-juicefs:/data + command: redis-server --port 6379 --requirepass ${REDIS_PASSWORD} --appendonly yes diff --git a/install/kubernetes/juicefs/juicefs-decive-of-k8s/juicefs-sc.yml b/install/kubernetes/juicefs/juicefs-decive-of-k8s/juicefs-sc.yml new file mode 100644 index 00000000..c41956c6 --- /dev/null +++ b/install/kubernetes/juicefs/juicefs-decive-of-k8s/juicefs-sc.yml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Secret +metadata: + name: juicefs-sc-secret + namespace: kube-system +type: Opaque +stringData: + name: "myjfs" + metaurl: "redis://:myredispassword@10.48.60.91:6382/1" + storage: "minio" + bucket: "http://10.48.60.91:9010/juicefs" + access-key: "root" + secret-key: "Dewe_2131" + # 如果需要设置 JuiceFS Mount Pod 的时区请将下一行的注释符号删除，默认为 UTC 时间。 + # envs: "{TZ: Asia/Shanghai}" +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: juicefs-sc +provisioner: csi.juicefs.com +reclaimPolicy: Retain +volumeBindingMode: Immediate +parameters: + csi.storage.k8s.io/node-publish-secret-name: juicefs-sc-secret + csi.storage.k8s.io/node-publish-secret-namespace: kube-system + csi.storage.k8s.io/provisioner-secret-name: juicefs-sc-secret + csi.storage.k8s.io/provisioner-secret-namespace: kube-system \ No newline at end of file diff --git a/install/kubernetes/juicefs/juicefs-decive-of-k8s/k8s_decive.yml b/install/kubernetes/juicefs/juicefs-decive-of-k8s/k8s_decive.yml new file mode 100644 index 00000000..94b3a8a2 --- /dev/null +++ b/install/kubernetes/juicefs/juicefs-decive-of-k8s/k8s_decive.yml @@ -0,0 +1,524 @@ +# DO NOT EDIT: generated by 'kustomize build' +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-controller-sa + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-node-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-external-node-service-role +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + - secrets + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - '*' +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-node-service-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: juicefs-csi-external-node-service-role +subjects: +- kind: ServiceAccount + name: juicefs-csi-node-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: juicefs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: juicefs-csi-controller-sa + namespace: kube-system +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: juicefs-csi-controller + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + serviceName: juicefs-csi-controller + template: + metadata: + labels: + app: juicefs-csi-controller + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + spec: + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --nodeid=$(NODE_NAME) + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: JUICEFS_MOUNT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: KUBELET_PORT + value: "10250" + - name: JUICEFS_MOUNT_PATH + value: /var/lib/juicefs/volume + - name: JUICEFS_CONFIG_PATH + value: /var/lib/juicefs/config + image: juicedata/juicefs-csi-driver:v0.17.1 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: juicefs-plugin + ports: + - containerPort: 9909 + name: healthz + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 512Mi + securityContext: + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /jfs + mountPropagation: Bidirectional + name: jfs-dir + - mountPath: /root/.juicefs + mountPropagation: Bidirectional + name: jfs-root-dir + - args: + - --csi-address=$(ADDRESS) + - --timeout=60s + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + name: csi-provisioner + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --health-port=$(HEALTH_PORT) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: HEALTH_PORT + value: "9909" + image: quay.io/k8scsi/livenessprobe:v1.1.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + priorityClassName: system-cluster-critical + serviceAccount: juicefs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: socket-dir + - hostPath: + path: /var/lib/juicefs/volume + type: DirectoryOrCreate + name: jfs-dir + - hostPath: + path: /var/lib/juicefs/config + type: DirectoryOrCreate + name: jfs-root-dir + volumeClaimTemplates: [] +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: juicefs-csi-node + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + template: + metadata: + labels: + app: juicefs-csi-node + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + spec: + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --nodeid=$(NODE_NAME) + - --v=5 + - --enable-manager=true + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: JUICEFS_MOUNT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: KUBELET_PORT + value: "10250" + - name: JUICEFS_MOUNT_PATH + value: /var/lib/juicefs/volume + - name: JUICEFS_CONFIG_PATH + value: /var/lib/juicefs/config + image: juicedata/juicefs-csi-driver:v0.17.1 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm /csi/csi.sock + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: juicefs-plugin + ports: + - containerPort: 9909 + name: healthz + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 512Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - mountPath: /jfs + mountPropagation: Bidirectional + name: jfs-dir + - mountPath: /root/.juicefs + mountPropagation: Bidirectional + name: jfs-root-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/csi-plugins/csi.juicefs.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=$(ADDRESS) + - --health-port=$(HEALTH_PORT) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: HEALTH_PORT + value: "9909" + image: quay.io/k8scsi/livenessprobe:v1.1.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + serviceAccount: juicefs-csi-node-sa + tolerations: + #- key: CriticalAddonsOnly 建议注释掉，直接忽略全部污点；这样可以让任何节点上都能挂载juicefs的csi驱动 + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/csi-plugins/csi.juicefs.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /var/lib/juicefs/volume + type: DirectoryOrCreate + name: jfs-dir + - hostPath: + path: /var/lib/juicefs/config + type: DirectoryOrCreate + name: jfs-root-dir +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: csi.juicefs.com +spec: + attachRequired: false + podInfoOnMount: false \ No newline at end of file diff --git a/install/kubernetes/juicefs/start_juicefs.sh b/install/kubernetes/juicefs/start_juicefs.sh new file mode 100644 index 00000000..a369dec1 --- /dev/null +++ b/install/kubernetes/juicefs/start_juicefs.sh @@ -0,0 +1,31 @@ +#记得提前修改.env中的ip地址，将其修改为自己的ip +source .env +#通过docker-compose启动用于元数据存储、块对象存储的redis和minio； +docker-compose up -d +#格式化文件系统,juicesfs支持将不同的redis database以及minio bucket格式化成不同的文件系统 +juicefs format \ + --storage minio \ + #--bucket http://10.48.60.91:9010/ \ + --bucket http://${JUICEFS_HOST_IP}:9010/juicefs \ + --access-key root \ + --secret-key Dewe_2131 \ + #"redis://:myredispassword@10.48.60.91:6382/" \ + "redis://:${REDIS_PASSWORD}@${JUICEFS_HOST_IP}:6382/1" \ + myjfs + +#安装juicefs的驱动 +for i in $(ls juicefs-decive-of-k8s/); do kubectl apply -f $i; done + +#将ip、reids密码、minio账号密码等改成.env文件中的 +kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"metaurl":"'"redis://:${REDIS_PASSWORD}@${JUICEFS_HOST_IP}:6382/1"'"}}' +kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"bucket":"'"http://${JUICEFS_HOST_IP}:9010/juicefs"'"}}' +kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"access-key":"'"${MINIO_ROOT_USER}"'"}}' +kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"secret-key":"'"${MINIO_ROOT_PASSWORD}"'"}}' + +#生成cube-studio所需的pv及pvc +for i in $(ls cube-pv-pvc-with-juicefs/); do kubectl apply -f $i; done + +#挂载到宿主机的/data/jfs目录，并指定redis的数据库1为元数据存储；这样方便调整、查看service pv、pipline pv中的内容 +juicefs mount -d "redis://:${REDIS_PASSWORD}@${JUICEFS_HOST_IP}:6382/1" /data/jfs +#卸载目录 +#juicefs umount -d /data/jfs \ No newline at end of file