添加template、pipeline/service初始化方法

This commit is contained in:
pengluan 2022-06-04 17:38:31 +08:00
parent 6cef0c5704
commit 02a120f2c2
8 changed files with 1923 additions and 1677 deletions

1
.gitignore vendored
View File

@ -7,3 +7,4 @@ myapp/test
#/myapp/static/appbuilder/vison/*
/myapp/static/appbuilder/mnt
/myapp/static/appbuilder/assets/
/images/web/target-detection/yolo/

View File

@ -6,7 +6,7 @@ cube是tme开源的一站式云原生机器学习平台目前主要包含
- 3、训练编排任务流编排在线拖拉拽开放的模板市场支持tf/pytorch/mxnet/spark/ray/horovod/kaldi/volcano等分布式计算/训练任务task的单节点debug分布式任务的批量优先级调度聚合日志任务运行资源监控报警定时调度支持补录忽略重试依赖并发限制定时任务算力的智能修正
- 4、超参搜索nnikatibray的超参搜索
- 5、推理服务tf/pytorch/onnx模型的推理服务serverless流量管控triton gpu推理加速依据gpu利用率/qps等指标的hpa能力虚拟化gpu虚拟显存等服务化能力
- 6、资源统筹多集群多项目组资源统筹联邦调度
- 6、资源统筹多集群多项目组资源统筹联邦调度,边缘计算
# 开源共建
@ -23,7 +23,7 @@ cube是tme开源的一站式云原生机器学习平台目前主要包含
| 自定义镜像 | 单机 | 完全自定义单机运行环境,可自由实现所有自定义单机功能 | 系统自带 |
| datax | 单机 | 异构数据源导入导出 | job-template/job/datax |
| xgb | 单机 | xgb模型训练 | job-template/job/xgb |
| deploy-service | 单机 | 部署云原生推理服务 | job-template/job/deploy-service |
| deploy-inferenceservice | 单机 | 部署云原生推理服务 | job-template/job/deploy-inferenceservice |
| ray | 分布式 | python ray框架 多机分布式功能,适用于超多文件在多机上的并发处理 | job-template/job/ray |
| ray-sklearn | 分布式 | 基于ray框架的sklearn支持算法多机分布式并行计算 | job-template/job/ray_sklearn |
| volcano | 分布式 | volcano框架的多机分布式可紫玉控制代码利用环境变量实现多机worker的工作与协同 | job-template/job/volcano |

View File

@ -356,6 +356,14 @@ NEO4J_AUTH=neo4j/admin
端口5775,16686
## 服务暴露:
- 1、域名暴露需要平台配置泛域名SERVICE_DOMAIN
- 2、ip暴露需要平台配置SERVICE_EXTERNAL_IP=[xx.xx.xx.xx]或者项目中配置SERVICE_EXTERNAL_IP=xx.xx.xx.xx ip需为集群中节点的ip或者clb的ip
# 推理服务
## 版本/域名/pod的关系
@ -435,3 +443,9 @@ K8S_POD_NAME=
```
## 服务暴露:
- 1、域名暴露需要平台配置泛域名SERVICE_DOMAIN
- 2、ip暴露需要平台配置SERVICE_EXTERNAL_IP=[xx.xx.xx.xx]或者项目中配置SERVICE_EXTERNAL_IP=xx.xx.xx.xx ip需为集群中节点的ip或者clb的ip

File diff suppressed because it is too large Load Diff

View File

@ -15,7 +15,7 @@ def init_db():
cursor = conn.cursor()
# 创建数据库的sql(如果数据库存在就不创建,防止异常)
sql = "CREATE DATABASE IF NOT EXISTS kubeflow DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
sql = "CREATE DATABASE IF NOT EXISTS %s DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"%uri.database
# 执行创建数据库的sql
cursor.execute(sql)
conn.commit()

1725
myapp/init-job-template.json Normal file

File diff suppressed because one or more lines are too long

62
myapp/init-pipeline.json Normal file

File diff suppressed because one or more lines are too long

47
myapp/init-service.json Normal file
View File

@ -0,0 +1,47 @@
{
"mysql-ui": {
"project_name":"public",
"service_name":"mysql-ui",
"service_describe":"可视化编辑mysql数据库",
"image_name":"ai.tencentmusic.com/tme-public/phpmyadmin",
"command":"",
"env":"PMA_HOST=mysql-service.infra\nPMA_PORT=3306\nPMA_USER=root\nPMA_PASSWORD=admin",
"ports":"80"
},
"redis-ui": {
"project_name":"public",
"service_name":"redis-ui",
"service_describe":"可视化编辑redis数据库",
"image_name":"ai.tencentmusic.com/tme-public/patrikx3:latest",
"command":"",
"env":"REDIS_NAME=default\nREDIS_HOST=redis-master.infra\nREDIS_PORT=6379\nREDIS_PASSWORD=admin",
"ports":"7843"
},
"mongo-express": {
"project_name":"public",
"service_name":"mongo-express",
"service_describe":"可视化编辑mongo数据库",
"image_name":"mongo-express:0.54.0",
"command":"",
"env":"ME_CONFIG_MONGODB_SERVER=xx.xx.xx.xx\nME_CONFIG_MONGODB_PORT=xx\nME_CONFIG_MONGODB_ENABLE_ADMIN=true\nME_CONFIG_MONGODB_ADMINUSERNAME=xx\nME_CONFIG_MONGODB_ADMINPASSWORD=xx\nME_CONFIG_MONGODB_AUTH_DATABASE=xx\nME_CONFIG_MONGODB_AUTH_USERNAME=xx\nME_CONFIG_MONGODB_AUTH_PASSWORD=xx\nVCAP_APP_HOST=0.0.0.0\nVCAP_APP_PORT=8081\nME_CONFIG_OPTIONS_EDITORTHEME=ambiance",
"ports":"8081"
},
"neo4j": {
"project_name":"public",
"service_name":"neo4j",
"service_describe":"可视化编辑图数据库neo4j",
"image_name":"ai.tencentmusic.com/tme-public/neo4j:4.4",
"command":"",
"env":"NEO4J_AUTH=neo4j/admin",
"ports":"7474,7687"
},
"jaeger": {
"project_name":"public",
"service_name":"jaeger",
"service_describe":"jaeger链路追踪",
"image_name":"jaegertracing/all-in-one:1.29",
"command":"",
"env":"",
"ports":"5775,16686"
}
}