mirror/cube-studio
2
0
Fork 0
mirror of https://github.com/tencentmusic/cube-studio.git synced 2025-02-11 14:34:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix readme

Browse Source
This commit is contained in:
FerdinandWard 2022-08-18 15:16:10 +08:00
parent 5116241ede
commit 0c1968fe45
13 changed files with 353 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
images/serving/onnxruntime/client.py
View File

@ -15,7 +15,7 @@ headers={
}
# onnxruntime
SERVER_URL = 'http://onnxruntime-resnet50-202112281143.kfserving/v1/models/resnet50'
SERVER_URL = 'http://onnxruntime-resnet50-202112281143/v1/models/resnet50'
IMAGE_PATH = 'smallcat.jpg'
files = {'data': open(IMAGE_PATH, 'rb')}
response = requests.post(SERVER_URL, data=files,headers=headers)

22
job-template/job/datax/README.md
View File

@ -2,3 +2,25 @@
镜像：ccr.ccs.tencentyun.com/cube-studio/datax
参数为job.json文件，格式可参考[datax github官网](https://github.com/alibaba/DataX)
参数
```bash
{
"参数": {
"-f": {
"type": "str",
"item_type": "str",
"label": "job.json文件地址，<a target='_blank' href='https://github.com/alibaba/DataX'>书写格式参考</a>",
"require": 1,
"choice": [],
"range": "",
"default": "/usr/local/datax/job/job.json",
"placeholder": "",
"describe": "job.json文件地址，<a target='_blank' href='https://github.com/alibaba/DataX'>书写格式参考</a>",
"editable": 1,
"condition": "",
"sub_args": {}
}
}
}
```

135
job-template/job/deploy-service/README.md
View File

@ -3,40 +3,12 @@
启动参数：
```bash
{
"shell": {
"--project_name": {
"type": "str",
"item_type": "str",
"label": "部署项目名",
"require": 1,
"choice": [],
"range": "",
"default": "public",
"placeholder": "",
"describe": "部署项目名",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--service_type": {
"type": "str",
"item_type": "str",
"label": "推理服务类型",
"require": 0,
"choice": ['service','tfserving','torch-server','onnxruntime','triton-server'],
"range": "",
"default": "service",
"placeholder": "",
"describe": "推理服务类型",
"editable": 1,
"condition": "",
"sub_args": {}
},
"模型信息": {
"--label": {
"type": "str",
"item_type": "str",
"label": "推理服务描述",
"require": 0,
"label": "中文描述描述",
"require": 1,
"choice": [],
"range": "",
"default": "demo推理服务",
@ -50,7 +22,7 @@
"type": "str",
"item_type": "str",
"label": "模型名",
"require": 0,
"require": 1,
"choice": [],
"range": "",
"default": "",
@ -64,7 +36,7 @@
"type": "str",
"item_type": "str",
"label": "模型版本号",
"require": 0,
"require": 1,
"choice": [],
"range": "",
"default": "v2022.10.01.1",
@ -87,12 +59,34 @@
"editable": 1,
"condition": "",
"sub_args": {}
}
},
"部署信息": {
"--service_type": {
"type": "str",
"item_type": "str",
"label": "推理服务类型",
"require": 1,
"choice": [
"serving",
"tfserving",
"torch-server",
"onnxruntime",
"triton-server"
],
"range": "",
"default": "service",
"placeholder": "",
"describe": "推理服务类型",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--images": {
"type": "str",
"item_type": "str",
"label": "推理服务镜像",
"require": 0,
"require": 1,
"choice": [],
"range": "",
"default": "",
@ -102,20 +96,6 @@
"condition": "",
"sub_args": {}
},
"--replicas": {
"type": "str",
"item_type": "str",
"label": "pod副本数",
"require": 0,
"choice": [],
"range": "",
"default": "1",
"placeholder": "",
"describe": "pod副本数",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--working_dir": {
"type": "str",
"item_type": "str",
@ -125,7 +105,7 @@
"range": "",
"default": "",
"placeholder": "",
"describe": "推理容器工作目录",
"describe": "推理容器工作目录,个人工作目录/mnt/$username",
"editable": 1,
"condition": "",
"sub_args": {}
@ -186,7 +166,62 @@
"condition": "",
"sub_args": {}
},
"--replicas": {
"type": "str",
"item_type": "str",
"label": "pod副本数",
"require": 1,
"choice": [],
"range": "",
"default": "1",
"placeholder": "",
"describe": "pod副本数",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--resource_memory": {
"type": "str",
"item_type": "str",
"label": "每个pod占用内存",
"require": 1,
"choice": [],
"range": "",
"default": "2G",
"placeholder": "",
"describe": "每个pod占用内存",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--resource_cpu": {
"type": "str",
"item_type": "str",
"label": "每个pod占用cpu",
"require": 1,
"choice": [],
"range": "",
"default": "2",
"placeholder": "",
"describe": "每个pod占用cpu",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--resource_gpu": {
"type": "str",
"item_type": "str",
"label": "每个pod占用gpu",
"require": 1,
"choice": [],
"range": "",
"default": "0",
"placeholder": "",
"describe": "每个pod占用gpu",
"editable": 1,
"condition": "",
"sub_args": {}
}
}
}
```

2
job-template/job/deploy-service/launcher.py
View File

@ -13,7 +13,7 @@ import os
KFJ_CREATOR = os.getenv('KFJ_CREATOR', 'admin')
KFJ_TASK_PROJECT_NAME = os.getenv('KFJ_TASK_PROJECT_NAME','public')
host = os.getenv('HOST','http://kubeflow-dashboard.infra').strip('/')
host = os.getenv('HOST',os.getenv('KFJ_MODEL_REPO_API_URL','http://kubeflow-dashboard.infra')).strip('/')
@pysnooper.snoop()
def deploy(**kwargs):

2
job-template/job/hadoop/README.md
View File

@ -6,7 +6,7 @@
参数
```bash
{
"shell": {
"参数": {
"--command": {
"type": "str",
"item_type": "str",

70
job-template/job/horovod/README.md
View File

@ -5,6 +5,48 @@ k8s账号: kubeflow-pipeline
```bash
{
"参数": {
"--work_images": {
"type": "str",
"item_type": "str",
"label": "worker的运行镜像，直接运行你代码的环境镜像 <a target='_blank' href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"require": 1,
"choice": [],
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/horovod:20210401",
"placeholder": "",
"describe": "worker的运行镜像",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--working_dir": {
"type": "str",
"item_type": "str",
"label": "命令的启动目录",
"require": 1,
"choice": [],
"range": "",
"default": "/mnt/xxx/horovod/",
"placeholder": "",
"describe": "命令的启动目录",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--command": {
"type": "str",
"item_type": "str",
"label": "训练启动命令",
"require": 1,
"choice": [],
"range": "",
"default": "python /mnt/admin/demo.py",
"placeholder": "",
"describe": "训练启动命令",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--num_worker": {
"type": "str",
"item_type": "str",
@ -18,34 +60,6 @@ k8s账号: kubeflow-pipeline
"editable": 1,
"condition": "",
"sub_args": {}
},
"--command": {
"type": "str",
"item_type": "str",
"label": "训练启动命令",
"require": 1,
"choice": [],
"range": "",
"default": "python /horovod/examples/tensorflow2/tensorflow2_mnist.py",
"placeholder": "",
"describe": "训练启动命令",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--work_images": {
"type": "str",
"item_type": "str",
"label": "worker的运行镜像",
"require": 1,
"choice": [],
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/horovod:20210401",
"placeholder": "",
"describe": "worker的运行镜像",
"editable": 1,
"condition": "",
"sub_args": {}
}
}
}

6
job-template/job/kaldi_distributed_on_volcanojob/README.md
View File

@ -12,7 +12,7 @@ TASK_RESOURCE_GPU=0
启动参数：
```bash
{
"shell": {
"参数": {
"--working_dir": {
"type": "str",
"item_type": "str",
@ -20,7 +20,7 @@ TASK_RESOURCE_GPU=0
"require": 1,
"choice": [],
"range": "",
"default": "",
"default": "/mnt/xx",
"placeholder": "启动目录",
"describe": "启动目录",
"editable": 1,
@ -64,7 +64,7 @@ TASK_RESOURCE_GPU=0
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/kaldi_distributed_worker:v1",
"placeholder": "",
"describe": "worker镜像，直接运行你代码的环境镜像 <a href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"describe": "worker镜像，直接运行你代码的环境镜像 <a target='_blank' href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"editable": 1,
"condition": "",
"sub_args": {}

30
job-template/job/model_offline_predict/README.md
View File

@ -12,7 +12,21 @@ TASK_RESOURCE_GPU=0
启动参数：
```bash
{
"shell": {
"参数": {
"--image": {
"type": "str",
"item_type": "str",
"label": "",
"require": 1,
"choice": [],
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda10.1-cudnn7-python3.6",
"placeholder": "",
"describe": "worker镜像，直接运行你代码的环境镜像<a target='_blank' href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--working_dir": {
"type": "str",
"item_type": "str",
@ -54,20 +68,6 @@ TASK_RESOURCE_GPU=0
"editable": 1,
"condition": "",
"sub_args": {}
},
"--image": {
"type": "str",
"item_type": "str",
"label": "",
"require": 1,
"choice": [],
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda10.1-cudnn7-python3.6",
"placeholder": "",
"describe": "worker镜像，直接运行你代码的环境镜像<a href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"editable": 1,
"condition": "",
"sub_args": {}
}
}
}

87
job-template/job/mxnet/README.md
View File

@ -0,0 +1,87 @@
镜像：ccr.ccs.tencentyun.com/cube-studio/mxnet:20221010
环境变量
```bash
NO_RESOURCE_CHECK=true
TASK_RESOURCE_CPU=2
TASK_RESOURCE_MEMORY=4G
TASK_RESOURCE_GPU=0
```
账号：kubeflow-pipeline
启动参数：
```bash
{
"参数": {
"--image": {
"type": "str",
"item_type": "str",
"label": "worker镜像，直接运行你代码的环境镜像",
"require": 1,
"choice": [],
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda10.1-cudnn7-python3.6",
"placeholder": "",
"describe": "worker镜像，直接运行你代码的环境镜像 <a target='_blank' href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--working_dir": {
"type": "str",
"item_type": "str",
"label": "命令的启动目录",
"require": 1,
"choice": [],
"range": "",
"default": "/mnt/xxx/mxnet/",
"placeholder": "",
"describe": "命令的启动目录",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--command": {
"type": "str",
"item_type": "str",
"label": "启动命令，例如 python3 xxx.py",
"require": 1,
"choice": [],
"range": "",
"default": "",
"placeholder": "启动命令，例如 python3 xxx.py",
"describe": "启动命令，例如 python3 xxx.py",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--num_worker": {
"type": "str",
"item_type": "str",
"label": "分布式训练worker的数目",
"require": 1,
"choice": [],
"range": "",
"default": "3",
"placeholder": "分布式训练worker的数目",
"describe": "分布式训练worker的数目",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--num_ps": {
"type": "str",
"item_type": "str",
"label": "分布式训练ps的数目",
"require": 1,
"choice": [],
"range": "",
"default": "0",
"placeholder": "分布式训练ps的数目",
"describe": "分布式训练ps的数目",
"editable": 1,
"condition": "",
"sub_args": {}
}
}
}
```

4
job-template/job/ner/Dockerfile
View File

@ -7,5 +7,5 @@ RUN python3 -m pip install sklearn_crfsuite -i https://pypi.tuna.tsinghua.edu.cn
RUN python3 -m pip install minio -i https://pypi.tuna.tsinghua.edu.cn/simple
RUN python3 -m pip install tqdm -i https://pypi.tuna.tsinghua.edu.cn/simple
RUN python3 -m pip install transformers -i https://pypi.tuna.tsinghua.edu.cn/simple
ENTRYPOINT ["python3", "/NER/main4argo.py"]
WORKDIR /NER
ENTRYPOINT ["python3", "main4argo.py"]

4
job-template/job/ner/README.md
View File

@ -1,4 +1,4 @@
# download_minio 模板
镜像：ccr.ccs.tencentyun.com/cube-studio/ner:20220812
启动参数：
```bash
@ -35,7 +35,7 @@
"sub_args": {}
},
"--epochs": {
"type": "int",
"type": "str",
"item_type": "str",
"label": "参数3",
"require": 1,

86
job-template/job/paddle/README.md
View File

@ -0,0 +1,86 @@
镜像:ccr.ccs.tencentyun.com/cube-studio/paddle:20221010
账号:kubeflow-pipeline
环境变量:
```bash
NO_RESOURCE_CHECK=true
TASK_RESOURCE_CPU=2
TASK_RESOURCE_MEMORY=4G
TASK_RESOURCE_GPU=0
```
启动参数：
```bash
{
"参数": {
"--image": {
"type": "str",
"item_type": "str",
"label": "worker镜像，直接运行你代码的环境镜像",
"require": 1,
"choice": [],
"range": "",
"default": "ccr.ccs.tencentyun.com/cube-studio/ubuntu-gpu:cuda10.1-cudnn7-python3.6",
"placeholder": "",
"describe": "worker镜像，直接运行你代码的环境镜像 <a target='_blank' href='https://github.com/tencentmusic/cube-studio/tree/master/images'>基础镜像</a>",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--working_dir": {
"type": "str",
"item_type": "str",
"label": "命令的启动目录",
"require": 1,
"choice": [],
"range": "",
"default": "/mnt/xxx/paddlejob/",
"placeholder": "",
"describe": "命令的启动目录",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--command": {
"type": "str",
"item_type": "str",
"label": "启动命令，例如 python3 xxx.py",
"require": 1,
"choice": [],
"range": "",
"default": "",
"placeholder": "启动命令，例如 python3 xxx.py",
"describe": "启动命令，例如 python3 xxx.py",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--num_worker": {
"type": "str",
"item_type": "str",
"label": "分布式训练worker的数目",
"require": 1,
"choice": [],
"range": "",
"default": "3",
"placeholder": "分布式训练worker的数目",
"describe": "分布式训练worker的数目",
"editable": 1,
"condition": "",
"sub_args": {}
},
"--num_ps": {
"type": "str",
"item_type": "str",
"label": "分布式训练ps的数目",
"require": 1,
"choice": [],
"range": "",
"default": "0",
"placeholder": "分布式训练ps的数目",
"describe": "分布式训练ps的数目",
"editable": 1,
"condition": "",
"sub_args": {}
}
}
}
```

12
job-template/job/spark/README.md
View File

@ -3,7 +3,7 @@
参数
```bash
{
"shell": {
"参数": {
"--image": {
"type": "str",
"item_type": "str",
@ -55,12 +55,12 @@
"type": "str",
"item_type": "str",
"label": "Java/Scala类名",
"require": 1,
"require": 0,
"choice": [],
"range": "",
"default": "",
"placeholder": "",
"describe": "Java/Scala类名",
"describe": "Java/Scala类名，其他语言下不填",
"editable": 1,
"condition": "",
"sub_args": {}
@ -83,7 +83,7 @@
"type": "str",
"item_type": "str",
"label": "代码参数",
"require": 1,
"require": 0,
"choice": [],
"range": "",
"default": "",
@ -97,7 +97,7 @@
"type": "text",
"item_type": "str",
"label": "spark配置",
"require": 1,
"require": 0,
"choice": [],
"range": "",
"default": "",
@ -111,7 +111,7 @@
"type": "text",
"item_type": "str",
"label": "hadoop配置，每行一个配置，xx=yy",
"require": 1,
"require": 0,
"choice": [],
"range": "",
"default": "",