mirror of
https://github.com/tencentmusic/cube-studio.git
synced 2025-01-30 14:09:48 +08:00
Merge branch 'master' of https://github.com/tencentmusic/cube-studio
This commit is contained in:
cdllp2
commit
fdff7239f2
4
install/kubernetes/juicefs/.env
Normal file
4
install/kubernetes/juicefs/.env
Normal file
@ -0,0 +1,4 @@
|
||||
JUICEFS_HOST_IP=10.48.60.91
|
||||
REDIS_PASSWORD=myredispassword
|
||||
MINIO_ROOT_USER=root
|
||||
MINIO_ROOT_PASSWORD=Dewe_2131
|
||||
3
install/kubernetes/juicefs/README.md
Normal file
3
install/kubernetes/juicefs/README.md
Normal file
@ -0,0 +1,3 @@
|
||||
### 如何以juicefs作为cube-stuido的训练、部署pv的共享目录：
|
||||
- 1，修改目录(juicefs)中的.env文件，将JUICEFS_HOST_IP改为自己部署cube-studio的节点的ip地址；
|
||||
- 2，在控制台执行sh start_juicefs.sh
|
||||
@ -0,0 +1,88 @@
|
||||
# 平台上传需要
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: infra-kubeflow-global-pv
|
||||
labels:
|
||||
infra-pvname: infra-kubeflow-global-pv
|
||||
spec:
|
||||
capacity:
|
||||
storage: 100Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: infra-kubeflow-global-pv
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/global
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-global-pvc
|
||||
namespace: infra
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
infra-pvname: infra-kubeflow-global-pv
|
||||
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: infra-kubeflow
|
||||
labels:
|
||||
infra-pvname: infra-kubeflow
|
||||
spec:
|
||||
capacity:
|
||||
storage: 1Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: infra-kubeflow
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: infra-kubeflow
|
||||
namespace: infra
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
infra-pvname: infra-kubeflow
|
||||
|
||||
@ -0,0 +1,90 @@
|
||||
|
||||
# 模型训练
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: jupyter-kubeflow-user-workspace
|
||||
labels:
|
||||
jupyter-pvname: jupyter-kubeflow-user-workspace
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: jupyter-kubeflow-user-workspace
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/workspace
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-user-workspace
|
||||
namespace: jupyter
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
jupyter-pvname: jupyter-kubeflow-user-workspace
|
||||
|
||||
|
||||
# 模型归档
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: jupyter-kubeflow-archives
|
||||
labels:
|
||||
jupyter-pvname: jupyter-kubeflow-archives
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: jupyter-kubeflow-archives
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/archives
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-archives
|
||||
namespace: jupyter
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
jupyter-pvname: jupyter-kubeflow-archives
|
||||
|
||||
@ -0,0 +1,45 @@
|
||||
|
||||
|
||||
# 模型训练
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: katib-kubeflow-user-workspace
|
||||
labels:
|
||||
katib-pvname: katib-kubeflow-user-workspace
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: katib-kubeflow-user-workspace
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/workspace
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-user-workspace
|
||||
namespace: katib
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
katib-pvname: katib-kubeflow-user-workspace
|
||||
@ -0,0 +1,44 @@
|
||||
# 模型训练
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: kubeflow-kubeflow-user-workspace
|
||||
labels:
|
||||
kubeflow-pvname: kubeflow-kubeflow-user-workspace
|
||||
spec:
|
||||
# storageClassName: pipeline
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: kubeflow-kubeflow-user-workspace
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/workspace
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-user-workspace
|
||||
namespace: kubeflow
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
kubeflow-pvname: kubeflow-kubeflow-user-workspace
|
||||
@ -0,0 +1,136 @@
|
||||
# 训练调用需要
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: pipeline-kubeflow-global-pv
|
||||
labels:
|
||||
pipeline-pvname: pipeline-kubeflow-global-pv
|
||||
spec:
|
||||
capacity:
|
||||
storage: 100Gi
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: pipeline-kubeflow-global-pv
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/global
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-global-pvc
|
||||
namespace: pipeline
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
pipeline-pvname: pipeline-kubeflow-global-pv
|
||||
|
||||
|
||||
# 模型训练
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: pipeline-kubeflow-user-workspace
|
||||
labels:
|
||||
pipeline-pvname: pipeline-kubeflow-user-workspace
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
volumeMode: Filesystem
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: pipeline-kubeflow-user-workspace
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/workspace
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-user-workspace
|
||||
namespace: pipeline
|
||||
spec:
|
||||
storageClassName: ""
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
pipeline-pvname: pipeline-kubeflow-user-workspace
|
||||
|
||||
# 模型归档
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: pipeline-kubeflow-archives
|
||||
labels:
|
||||
pipeline-pvname: pipeline-kubeflow-archives
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
volumeMode: Filesystem
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: pipeline-kubeflow-archives
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/archives
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-archives
|
||||
namespace: pipeline
|
||||
spec:
|
||||
storageClassName: ""
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
pipeline-pvname: pipeline-kubeflow-archives
|
||||
@ -0,0 +1,87 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: service-kubeflow-user-workspace
|
||||
labels:
|
||||
service-pvname: service-kubeflow-user-workspace
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
volumeMode: Filesystem
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: service-kubeflow-user-workspace
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/workspace
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: service-kubeflow-archives
|
||||
labels:
|
||||
service-pvname: service-kubeflow-archives
|
||||
spec:
|
||||
capacity:
|
||||
storage: 500Gi
|
||||
volumeMode: Filesystem
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: csi.juicefs.com
|
||||
volumeHandle: service-kubeflow-archives
|
||||
fsType: juicefs
|
||||
nodePublishSecretRef:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
volumeAttributes:
|
||||
juicefs/mount-cpu-limit: 5000m
|
||||
juicefs/mount-memory-limit: 5Gi
|
||||
juicefs/mount-cpu-request: 1m
|
||||
juicefs/mount-memory-request: 1Mi
|
||||
mountOptions:
|
||||
- subdir=kubeflow/pipeline/archives
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-user-workspace
|
||||
namespace: service
|
||||
spec:
|
||||
storageClassName: ""
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
service-pvname: service-kubeflow-user-workspace
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kubeflow-archives
|
||||
namespace: service
|
||||
spec:
|
||||
storageClassName: ""
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 500Gi
|
||||
selector:
|
||||
matchLabels:
|
||||
service-pvname: service-kubeflow-archives
|
||||
35
install/kubernetes/juicefs/docker-compose.yml
Normal file
35
install/kubernetes/juicefs/docker-compose.yml
Normal file
@ -0,0 +1,35 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
minio-juicefs:
|
||||
container_name: minio-juicefs
|
||||
restart: always
|
||||
image: bitnami/minio:2022.9.17
|
||||
#ports: #使用host模式时不需要
|
||||
# - '9010:9000'
|
||||
# - '9011:9001'
|
||||
volumes:
|
||||
- /data/allvolumes/minio/minio_data:/data
|
||||
#- /data/allvolumes/minio/minio_config:/root/.minio
|
||||
command: minio server --address ${JUICEFS_HOST_IP}:9010 --console-address ${JUICEFS_HOST_IP}:9011 /data
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: ${MINIO_ROOT_USER} #用于外部接口调用
|
||||
MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER} #用于登录管理页面
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_DEFAULT_BUCKETS: "juicefs:policy" #默认得bucket
|
||||
network_mode: host
|
||||
|
||||
redis-juicefs:
|
||||
image: redis:7.0.4-alpine3.16
|
||||
container_name: redis-juicefs
|
||||
restart: always
|
||||
#expose:
|
||||
# - 6379
|
||||
ports: #使用host模式时不需要
|
||||
- 6382:6379
|
||||
restart: always
|
||||
#network_mode: host
|
||||
volumes:
|
||||
- /data/allvolumes/redis-juicefs:/data
|
||||
command: redis-server --port 6379 --requirepass ${REDIS_PASSWORD} --appendonly yes
|
||||
@ -0,0 +1,28 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: juicefs-sc-secret
|
||||
namespace: kube-system
|
||||
type: Opaque
|
||||
stringData:
|
||||
name: "myjfs"
|
||||
metaurl: "redis://:myredispassword@10.48.60.91:6382/1"
|
||||
storage: "minio"
|
||||
bucket: "http://10.48.60.91:9010/juicefs"
|
||||
access-key: "root"
|
||||
secret-key: "Dewe_2131"
|
||||
# 如果需要设置 JuiceFS Mount Pod 的时区请将下一行的注释符号删除，默认为 UTC 时间。
|
||||
# envs: "{TZ: Asia/Shanghai}"
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: juicefs-sc
|
||||
provisioner: csi.juicefs.com
|
||||
reclaimPolicy: Retain
|
||||
volumeBindingMode: Immediate
|
||||
parameters:
|
||||
csi.storage.k8s.io/node-publish-secret-name: juicefs-sc-secret
|
||||
csi.storage.k8s.io/node-publish-secret-namespace: kube-system
|
||||
csi.storage.k8s.io/provisioner-secret-name: juicefs-sc-secret
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: kube-system
|
||||
524
install/kubernetes/juicefs/juicefs-decive-of-k8s/k8s_decive.yml
Normal file
524
install/kubernetes/juicefs/juicefs-decive-of-k8s/k8s_decive.yml
Normal file
@ -0,0 +1,524 @@
|
||||
# DO NOT EDIT: generated by 'kustomize build'
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-controller-sa
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-node-sa
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-external-node-service-role
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- pods/log
|
||||
- secrets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- patch
|
||||
- watch
|
||||
- apiGroups:
|
||||
- batch
|
||||
resources:
|
||||
- jobs
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- patch
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes/proxy
|
||||
verbs:
|
||||
- '*'
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- persistentvolumes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-external-provisioner-role
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- persistentvolumes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- update
|
||||
- apiGroups:
|
||||
- storage.k8s.io
|
||||
resources:
|
||||
- storageclasses
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- events
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- patch
|
||||
- apiGroups:
|
||||
- storage.k8s.io
|
||||
resources:
|
||||
- csinodes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- create
|
||||
- update
|
||||
- patch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- pods/log
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- patch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- batch
|
||||
resources:
|
||||
- jobs
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- patch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- endpoints
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- patch
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-node-service-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: juicefs-csi-external-node-service-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: juicefs-csi-node-sa
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-provisioner-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: juicefs-external-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: juicefs-csi-controller-sa
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: controller
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-controller
|
||||
namespace: kube-system
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: juicefs-csi-controller
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
serviceName: juicefs-csi-controller
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: juicefs-csi-controller
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- --endpoint=$(CSI_ENDPOINT)
|
||||
- --logtostderr
|
||||
- --nodeid=$(NODE_NAME)
|
||||
- --v=5
|
||||
env:
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: JUICEFS_MOUNT_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: HOST_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.hostIP
|
||||
- name: KUBELET_PORT
|
||||
value: "10250"
|
||||
- name: JUICEFS_MOUNT_PATH
|
||||
value: /var/lib/juicefs/volume
|
||||
- name: JUICEFS_CONFIG_PATH
|
||||
value: /var/lib/juicefs/config
|
||||
image: juicedata/juicefs-csi-driver:v0.17.1
|
||||
livenessProbe:
|
||||
failureThreshold: 5
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: healthz
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
name: juicefs-plugin
|
||||
ports:
|
||||
- containerPort: 9909
|
||||
name: healthz
|
||||
protocol: TCP
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- SYS_ADMIN
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
||||
name: socket-dir
|
||||
- mountPath: /jfs
|
||||
mountPropagation: Bidirectional
|
||||
name: jfs-dir
|
||||
- mountPath: /root/.juicefs
|
||||
mountPropagation: Bidirectional
|
||||
name: jfs-root-dir
|
||||
- args:
|
||||
- --csi-address=$(ADDRESS)
|
||||
- --timeout=60s
|
||||
- --v=5
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
||||
image: quay.io/k8scsi/csi-provisioner:v1.6.0
|
||||
name: csi-provisioner
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
||||
name: socket-dir
|
||||
- args:
|
||||
- --csi-address=$(ADDRESS)
|
||||
- --health-port=$(HEALTH_PORT)
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: HEALTH_PORT
|
||||
value: "9909"
|
||||
image: quay.io/k8scsi/livenessprobe:v1.1.0
|
||||
name: liveness-probe
|
||||
volumeMounts:
|
||||
- mountPath: /csi
|
||||
name: socket-dir
|
||||
priorityClassName: system-cluster-critical
|
||||
serviceAccount: juicefs-csi-controller-sa
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
volumes:
|
||||
- emptyDir: {}
|
||||
name: socket-dir
|
||||
- hostPath:
|
||||
path: /var/lib/juicefs/volume
|
||||
type: DirectoryOrCreate
|
||||
name: jfs-dir
|
||||
- hostPath:
|
||||
path: /var/lib/juicefs/config
|
||||
type: DirectoryOrCreate
|
||||
name: jfs-root-dir
|
||||
volumeClaimTemplates: []
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: node
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: juicefs-csi-node
|
||||
namespace: kube-system
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: juicefs-csi-node
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: juicefs-csi-node
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- --endpoint=$(CSI_ENDPOINT)
|
||||
- --logtostderr
|
||||
- --nodeid=$(NODE_NAME)
|
||||
- --v=5
|
||||
- --enable-manager=true
|
||||
env:
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix:/csi/csi.sock
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: JUICEFS_MOUNT_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: HOST_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.hostIP
|
||||
- name: KUBELET_PORT
|
||||
value: "10250"
|
||||
- name: JUICEFS_MOUNT_PATH
|
||||
value: /var/lib/juicefs/volume
|
||||
- name: JUICEFS_CONFIG_PATH
|
||||
value: /var/lib/juicefs/config
|
||||
image: juicedata/juicefs-csi-driver:v0.17.1
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- rm /csi/csi.sock
|
||||
livenessProbe:
|
||||
failureThreshold: 5
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: healthz
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
name: juicefs-plugin
|
||||
ports:
|
||||
- containerPort: 9909
|
||||
name: healthz
|
||||
protocol: TCP
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/kubelet
|
||||
mountPropagation: Bidirectional
|
||||
name: kubelet-dir
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
- mountPath: /dev
|
||||
name: device-dir
|
||||
- mountPath: /jfs
|
||||
mountPropagation: Bidirectional
|
||||
name: jfs-dir
|
||||
- mountPath: /root/.juicefs
|
||||
mountPropagation: Bidirectional
|
||||
name: jfs-root-dir
|
||||
- args:
|
||||
- --csi-address=$(ADDRESS)
|
||||
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
|
||||
- --v=5
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: DRIVER_REG_SOCK_PATH
|
||||
value: /var/lib/kubelet/csi-plugins/csi.juicefs.com/csi.sock
|
||||
image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0
|
||||
name: node-driver-registrar
|
||||
volumeMounts:
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
- mountPath: /registration
|
||||
name: registration-dir
|
||||
- args:
|
||||
- --csi-address=$(ADDRESS)
|
||||
- --health-port=$(HEALTH_PORT)
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: HEALTH_PORT
|
||||
value: "9909"
|
||||
image: quay.io/k8scsi/livenessprobe:v1.1.0
|
||||
name: liveness-probe
|
||||
volumeMounts:
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
priorityClassName: system-node-critical
|
||||
serviceAccount: juicefs-csi-node-sa
|
||||
tolerations:
|
||||
#- key: CriticalAddonsOnly 建议注释掉，直接忽略全部污点；这样可以让任何节点上都能挂载juicefs的csi驱动
|
||||
operator: Exists
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet
|
||||
type: Directory
|
||||
name: kubelet-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/csi-plugins/csi.juicefs.com/
|
||||
type: DirectoryOrCreate
|
||||
name: plugin-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
name: registration-dir
|
||||
- hostPath:
|
||||
path: /dev
|
||||
type: Directory
|
||||
name: device-dir
|
||||
- hostPath:
|
||||
path: /var/lib/juicefs/volume
|
||||
type: DirectoryOrCreate
|
||||
name: jfs-dir
|
||||
- hostPath:
|
||||
path: /var/lib/juicefs/config
|
||||
type: DirectoryOrCreate
|
||||
name: jfs-root-dir
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: CSIDriver
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: juicefs-csi-driver
|
||||
app.kubernetes.io/name: juicefs-csi-driver
|
||||
app.kubernetes.io/version: master
|
||||
name: csi.juicefs.com
|
||||
spec:
|
||||
attachRequired: false
|
||||
podInfoOnMount: false
|
||||
31
install/kubernetes/juicefs/start_juicefs.sh
Normal file
31
install/kubernetes/juicefs/start_juicefs.sh
Normal file
@ -0,0 +1,31 @@
|
||||
#记得提前修改.env中的ip地址，将其修改为自己的ip
|
||||
source .env
|
||||
#通过docker-compose启动用于元数据存储、块对象存储的redis和minio；
|
||||
docker-compose up -d
|
||||
#格式化文件系统,juicesfs支持将不同的redis database以及minio bucket格式化成不同的文件系统
|
||||
juicefs format \
|
||||
--storage minio \
|
||||
#--bucket http://10.48.60.91:9010/<bucket> \
|
||||
--bucket http://${JUICEFS_HOST_IP}:9010/juicefs \
|
||||
--access-key root \
|
||||
--secret-key Dewe_2131 \
|
||||
#"redis://:myredispassword@10.48.60.91:6382/<database>" \
|
||||
"redis://:${REDIS_PASSWORD}@${JUICEFS_HOST_IP}:6382/1" \
|
||||
myjfs
|
||||
|
||||
#安装juicefs的驱动
|
||||
for i in $(ls juicefs-decive-of-k8s/); do kubectl apply -f $i; done
|
||||
|
||||
#将ip、reids密码、minio账号密码等改成.env文件中的
|
||||
kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"metaurl":"'"redis://:${REDIS_PASSWORD}@${JUICEFS_HOST_IP}:6382/1"'"}}'
|
||||
kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"bucket":"'"http://${JUICEFS_HOST_IP}:9010/juicefs"'"}}'
|
||||
kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"access-key":"'"${MINIO_ROOT_USER}"'"}}'
|
||||
kubectl patch Secret juicefs-sc-secret -n kube-system -p '{"stringData":{"secret-key":"'"${MINIO_ROOT_PASSWORD}"'"}}'
|
||||
|
||||
#生成cube-studio所需的pv及pvc
|
||||
for i in $(ls cube-pv-pvc-with-juicefs/); do kubectl apply -f $i; done
|
||||
|
||||
#挂载到宿主机的/data/jfs目录，并指定redis的数据库1为元数据存储；这样方便调整、查看service pv、pipline pv中的内容
|
||||
juicefs mount -d "redis://:${REDIS_PASSWORD}@${JUICEFS_HOST_IP}:6382/1" /data/jfs
|
||||
#卸载目录
|
||||
#juicefs umount -d /data/jfs
|
||||
Loading…
Reference in New Issue
Block a user