mirror of
https://github.com/tencentmusic/cube-studio.git
synced 2024-11-27 05:33:10 +08:00
add user pod monitoring
This commit is contained in:
FerdinandWard
parent
cab99b5a8e
commit
c96ae408e3
@ -28,7 +28,11 @@ from flask_appbuilder import ModelView,AppBuilder,expose,BaseView,has_access
|
||||
from myapp import app, appbuilder
|
||||
from flask import stream_with_context, request
|
||||
|
||||
resource_used = {
|
||||
node_resource_used = {
|
||||
"check_time": None,
|
||||
"data": {}
|
||||
}
|
||||
pipeline_resource_used={
|
||||
"check_time": None,
|
||||
"data": {}
|
||||
}
|
||||
@ -47,10 +51,12 @@ class Myapp(BaseMyappView):
|
||||
else:
|
||||
msg = 'Hello '+g.user.username+" !"
|
||||
|
||||
# 返回模板
|
||||
return self.render_template('hello.html', msg=msg)
|
||||
|
||||
@expose('/home')
|
||||
def home(self):
|
||||
# 返回模板
|
||||
return self.render_template('home.html')
|
||||
|
||||
|
||||
@ -478,6 +484,67 @@ class Myapp(BaseMyappView):
|
||||
"url": "/service_pipeline_modelview/api/"
|
||||
},
|
||||
]
|
||||
},
|
||||
# 应用市场
|
||||
{
|
||||
"name": 'aihub',
|
||||
"title": 'AIHub',
|
||||
"isMenu": True,
|
||||
# "isExpand": True,
|
||||
"icon": '<svg t="1660895494223" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2520" width="200" height="200"><path d="M735.9 66.5H288.1C176.4 66.5 85.5 157.3 85.5 269v486c0 111.7 90.9 202.6 202.6 202.6H736c111.7 0 202.6-90.9 202.6-202.6V269c-0.1-111.7-91-202.5-202.7-202.5zM884.8 755c0 82.1-66.8 148.9-148.9 148.9H288.1C206 903.8 139.2 837 139.2 755V269c0-82.1 66.8-148.9 148.9-148.9H736c82.1 0 148.9 66.8 148.9 148.9v486z" fill="#494949" p-id="2521"></path><path d="M651.2 344.5c-18.5 0-33.6 15-33.6 33.6 0 58.3-47.4 105.7-105.7 105.7-58.3 0-105.7-47.4-105.7-105.7 0-18.5-15-33.6-33.6-33.6-18.5 0-33.6 15-33.6 33.6 0 95.3 77.5 172.8 172.8 172.8s172.8-77.5 172.8-172.8c0.2-18.6-14.8-33.6-33.4-33.6z" fill="#494949" p-id="2522"></path></svg>',
|
||||
"children": [
|
||||
{
|
||||
"name": 'app_market',
|
||||
"title": '应用市场',
|
||||
"isMenu": True,
|
||||
"isExpand": True,
|
||||
"children": [
|
||||
{
|
||||
"name": 'dataset_market',
|
||||
"title": '数据集',
|
||||
"icon": '<svg t="1660895595231" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5135" width="200" height="200"><path d="M685.6 444l171.7 68.7-354.8 141.9-337.2-141.9 175-73.7-37.3-17-208.6 90.7 408.1 177.4 425.8-177.4-201.7-84-41 15.3z m0 230.7l171.7 68.7-354.8 141.9-337.2-141.9 175-73.7-37.4-17-208.5 90.7 408.1 177.4 425.8-177.4-201.7-84-41 15.3zM928.3 282L502.5 104.6 94.4 282l408.1 177.4L928.3 282z m-763 0l337.1-141.9L857.3 282 502.5 424 165.3 282z m586.1 17.8V282L503 388.5 254.5 282v17.7L503 406.2l248.4-106.4z" fill="" p-id="5136"></path></svg>',
|
||||
"menu_type": "api",
|
||||
"url": "/training_model_modelview/api/"
|
||||
},
|
||||
{
|
||||
"name": 'template_market',
|
||||
"title": '计算模板',
|
||||
"icon": '<svg t="1660896397077" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="12417" width="200" height="200"><path d="M815.1 380.1c-81.4-27.7-189-42.9-302.9-42.9h-7.3c30.1-39 61.1-72.7 91.3-99.2 26.5-23.2 51.4-39.7 72-47.7 14.4-5.6 26.2-6.7 32.4-3.2 6.7 3.9 11.8 16 13.9 33.3 3 24.4 0 57.5-8.7 95.7L776 332c10.6-46.9 14-87.4 9.9-120.4-3.5-28.7-14.9-67-49.4-86.9-33.1-19.1-92.5-24.4-187.8 59.1-45.6 39.9-92.3 94.2-135.8 157.3-29.3 2.4-57.6 5.8-84.7 10.2-4.9-16.7-8.9-32.9-12-48.4-6.9-34.6-8.7-64.4-5.3-86.2 2.4-15.2 7.2-26 13.4-29.6 10.2-5.9 46.5-1.7 110.5 56.4l48.4-53.3c-34.5-31.3-67.1-53.4-97-65.7-26-10.7-64.1-19.3-97.9 0.2-33.1 19.1-67.4 68-42.7 192.2 3.1 15.8 7.1 32.1 11.9 49-16.9 4.3-33 9-48.2 14.2C89.4 420.9 64.2 475 64.2 513.2c0 39.3 26.8 68.1 49.4 85.3 25.9 19.8 61.8 37.1 106.7 51.3l21.8-68.6c-83.7-26.5-105.8-56.1-105.8-68 0-7.1 6.9-16.8 18.9-26.4 17.2-13.8 44-27.1 77.3-38.5 15-5.1 31-9.7 47.8-13.8 9.7 25.7 21 52 33.6 78.6-17.6 37.2-32.6 74.1-44.6 109.4-21 62.2-32 118.9-31.8 163.8 0.2 38.9 9.2 90.3 51.1 114.5 14.7 8.5 30.1 11.6 45.1 11.6 19.7 0 38.4-5.5 53.3-11.6 30.1-12.4 62.9-34.8 97.5-66.4L436 781.2c-64.6 58.9-101.1 63.2-111.4 57.2-7.2-4.2-14.9-20.1-15.1-52.5-0.2-37.1 9.5-85.7 28-140.5 5.6-16.6 11.9-33.5 19-50.7 1.2 2 2.3 4 3.5 6.1 38.1 65.9 81.5 127.3 125.6 177.4 43.4 49.3 87 87.2 126 109.5 22.1 12.7 50.9 24.7 79.8 24.7 15.2 0 30.5-3.3 45-11.7 41.9-24.2 50.9-75.7 51.1-114.6 0.2-44.9-10.8-101.6-31.8-163.9-10.7-31.8-24-65-39.4-98.5L650.8 554c14.4 31.2 26.7 61.9 36.6 91.4 18.5 54.8 28.2 103.4 28 140.6-0.1 32.4-7.9 48.4-15.1 52.5-7.2 4.2-24.9 2.9-53-13.2-32.2-18.4-69.5-51.1-107.7-94.6-41-46.6-81.6-103.9-117.3-165.8-10-17.3-19.4-34.5-28.2-51.8 8.9-17.5 18.4-34.9 28.4-52.2 9.9-17.2 20.1-33.9 30.6-50.1 19.4-1 39.1-1.5 59.1-1.5 106.2 0 205.6 13.9 279.7 39.1 33.4 11.3 60.1 24.6 77.3 38.5 12 9.7 18.9 19.3 18.9 26.4 0 7.7-7.9 18.1-21.7 28.6-19.4 14.7-49.3 28.6-86.4 40.2l21.5 68.7c45.6-14.2 82.1-31.6 108.4-51.5 22.9-17.3 50.2-46.3 50.2-86 0.1-38.3-25.1-92.4-145-133.2z m-455 44.7c-1.3 2.2-2.5 4.4-3.8 6.6-1.5-3.6-2.9-7.2-4.4-10.8 3.8-0.6 7.7-1.1 11.6-1.6-1.1 1.8-2.2 3.8-3.4 5.8z" p-id="12418"></path><path d="M538.7 526.3m-90 0a90 90 0 1 0 180 0 90 90 0 1 0-180 0Z" p-id="12419"></path></svg>',
|
||||
"menu_type": "api",
|
||||
"url": "/inferenceservice_modelview/api/"
|
||||
},
|
||||
{
|
||||
"name": 'model_market',
|
||||
"title": '预训练模型',
|
||||
"icon": '<svg t="1660896421476" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="13420" width="200" height="200"><path d="M766.95254 567.536595c-11.798862 145.017572-133.214195 259.031297-281.272546 259.031297-155.869672 0-282.227136-126.357464-282.227137-282.227136 0-146.895143 112.22797-267.568719 255.617833-280.978433V182.086723C270.893377 195.712281 122.457524 352.688266 122.457524 544.340756c0 200.602334 162.620437 363.22247 363.22247 363.222469 192.809678 0 350.522903-150.234251 362.485229-340.026931h-81.212683z" fill="#515151" p-id="13421"></path><path d="M447.657457 182.086723h76.045073v362.254033h-76.045073z" fill="#515151" p-id="13422"></path><path d="M447.657457 506.299555h399.908099v76.045073H447.657457zM562.796158 135.143845h62.335828v296.947868h-62.335828z" fill="#515151" p-id="13423"></path><path d="M562.796158 400.908446h327.813239v62.335828H562.796158z" fill="#515151" p-id="13424"></path><path d="M847.973462 414.70981c0 16.840337-1.801709 33.259522-5.219087 49.077234h67.580502c2.648528-15.966425 4.032397-32.359721 4.032397-49.077234 0-164.438402-133.303302-297.741704-297.741704-297.741704-19.212813 0-38.000862 1.819771-56.200985 5.296754l2.371874 67.392956c17.272325-4.116386 35.296341-6.295596 53.829412-6.295597 127.769631-0.000301 231.347591 103.577659 231.347591 231.347591z" fill="#515151" p-id="13425"></path></svg>',
|
||||
"menu_type": "api",
|
||||
"url": "/inferenceservice_modelview/api/"
|
||||
},
|
||||
{
|
||||
"name": 'pipeline_market',
|
||||
"title": '算法任务流',
|
||||
"icon": '<svg t="1660896506028" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="13603" width="200" height="200"><path d="M576.364 706c59.702 0 108.1 48.398 108.1 108.1 0 59.702-48.398 108.1-108.1 108.1h-179.4c-59.702 0-108.1-48.398-108.1-108.1 0-59.702 48.398-108.1 108.1-108.1h179.4z m196.737-188.572c78.348 3.4 140.802 68.833 151.318 148.04 11.261 84.825-39.41 157.519-138.707 179.422l1.457 41.706a6.9 6.9 0 0 1-11.165 5.662l-82.04-64.617a6.9 6.9 0 0 1-0.367-10.53l77.321-70.17a6.9 6.9 0 0 1 11.533 4.869l1.123 32.004c61.365-17.221 88.357-59.32 81.565-110.477-6.833-51.47-46.297-93.259-93.207-96.091l-1.424-0.074-0.715-0.04c-16.15-1.081-28.566-14.87-27.86-31.128 0.716-16.498 14.67-29.292 31.168-28.576zM576.364 765.8h-179.4c-26.675 0-48.3 21.625-48.3 48.3 0 26.406 21.19 47.863 47.493 48.293l0.807 0.007h179.4c26.676 0 48.3-21.625 48.3-48.3 0-26.406-21.19-47.863-47.493-48.293l-0.807-0.007z m0-326.6c59.702 0 108.1 48.398 108.1 108.1 0 59.702-48.398 108.1-108.1 108.1h-179.4c-59.702 0-108.1-48.398-108.1-108.1 0-59.702 48.398-108.1 108.1-108.1h179.4zM261.236 260.804c0.706 16.258-11.71 30.047-27.86 31.128l-0.715 0.04c-47.534 2.063-87.729 44.175-94.631 96.165-5.815 43.8 13.134 80.958 57.218 101.55l1.02-29.118a6.9 6.9 0 0 1 11.532-4.868l77.321 70.169a6.9 6.9 0 0 1-0.367 10.53l-82.04 64.617a6.9 6.9 0 0 1-11.164-5.662l1.48-42.383C109.937 525.25 68.453 457.83 78.75 380.267c10.515-79.206 72.97-144.639 151.318-148.039 16.498-0.716 30.452 12.078 31.168 28.576zM576.364 499h-179.4c-26.675 0-48.3 21.625-48.3 48.3 0 26.406 21.19 47.863 47.493 48.293l0.807 0.007h179.4c26.676 0 48.3-21.625 48.3-48.3 0-26.406-21.19-47.863-47.493-48.293l-0.807-0.007z m0-345c59.702 0 108.1 48.398 108.1 108.1 0 59.702-48.398 108.1-108.1 108.1h-179.4c-59.702 0-108.1-48.398-108.1-108.1 0-59.702 48.398-108.1 108.1-108.1h179.4z m0 59.8h-179.4c-26.675 0-48.3 21.625-48.3 48.3 0 26.406 21.19 47.863 47.493 48.293l0.807 0.007h179.4c26.676 0 48.3-21.625 48.3-48.3 0-26.406-21.19-47.863-47.493-48.293l-0.807-0.007z" fill="#333333" p-id="13604"></path></svg>',
|
||||
"menu_type": "api",
|
||||
"url": "/inferenceservice_modelview/api/"
|
||||
},
|
||||
{
|
||||
"name": 'service_market',
|
||||
"title": '推理服务',
|
||||
"icon": '<svg t="1660896559384" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="16448" width="200" height="200"><path d="M485.376 931.84c-231.424 0-419.84-188.416-419.84-419.84s188.416-419.84 419.84-419.84c197.632 0 370.688 140.288 410.624 333.824 2.048 11.264 4.096 23.552 6.144 34.816 2.048 16.384-10.24 31.744-26.624 33.792-16.384 2.048-31.744-10.24-33.792-26.624-1.024-10.24-3.072-20.48-5.12-29.696-34.816-165.888-182.272-286.72-352.256-286.72-197.632 1.024-358.4 161.792-358.4 360.448S286.72 871.424 485.376 871.424c177.152 0 329.728-132.096 355.328-306.176 2.048-16.384 17.408-27.648 33.792-25.6 16.384 2.048 27.648 17.408 25.6 33.792C870.4 778.24 691.2 931.84 485.376 931.84z" p-id="16449"></path><path d="M95.232 758.784c-45.056 0-70.656-12.288-80.896-36.864-19.456-45.056 34.816-94.208 77.824-126.976 13.312-10.24 31.744-8.192 41.984 5.12 10.24 13.312 8.192 31.744-5.12 41.984-36.864 28.672-51.2 46.08-56.32 55.296 39.936 10.24 202.752-16.384 464.896-130.048 262.144-113.664 393.216-215.04 412.672-250.88-12.288-3.072-47.104-5.12-131.072 14.336-16.384 4.096-32.768-6.144-35.84-22.528-4.096-16.384 6.144-32.768 22.528-35.84 122.88-28.672 185.344-22.528 203.776 18.432 17.408 40.96-19.456 90.112-121.856 158.72-82.944 56.32-198.656 117.76-325.632 173.056-126.976 55.296-250.88 98.304-349.184 119.808-48.128 11.264-87.04 16.384-117.76 16.384z" p-id="16450"></path></svg>',
|
||||
"menu_type": "api",
|
||||
"url": "/inferenceservice_modelview/api/"
|
||||
},
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": 'business_market',
|
||||
"title": '需求市场',
|
||||
"icon": '<svg t="1660896350755" class="icon" viewBox="0 0 1075 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="11649" width="200" height="200"><path d="M563.2 998.64381H102.4c-14.140952 0-25.35619-11.215238-25.35619-25.356191V204.8c0-42.422857 34.620952-77.04381 77.043809-77.04381h358.4c42.422857 0 76.55619 34.620952 76.556191 77.04381v768a26.331429 26.331429 0 0 1-25.84381 25.84381z m-435.44381-51.68762h409.6V204.8c0-14.140952-11.702857-25.84381-25.35619-25.84381H153.6c-14.140952 0-25.84381 11.702857-25.84381 25.84381v742.15619z" p-id="11650"></path><path d="M870.4 998.64381h-307.2c-14.140952 0-25.84381-11.215238-25.84381-25.356191v-409.6c0-42.422857 34.620952-76.55619 77.04381-76.55619h204.8c42.422857 0 77.04381 34.620952 77.04381 76.55619v409.6c0 13.653333-11.702857 25.35619-25.84381 25.356191z m-281.84381-51.68762h256v-383.75619c0-14.140952-11.702857-25.84381-25.843809-25.84381h-204.8c-14.140952 0-25.84381 11.702857-25.84381 25.84381v383.75619z" p-id="11651"></path><path d="M435.44381 537.35619H204.8c-14.140952 0-25.35619-11.215238-25.35619-25.35619s11.215238-25.35619 25.35619-25.35619h230.64381c14.140952 0 25.35619 11.215238 25.35619 25.35619s-11.215238 25.35619-25.35619 25.35619z" p-id="11652"></path><path d="M435.44381 384.24381H204.8c-14.140952 0-25.35619-11.215238-25.35619-25.84381s11.215238-25.84381 25.35619-25.84381h230.64381c14.140952 0 25.35619 11.215238 25.35619 25.84381s-11.215238 25.84381-25.35619 25.84381z" p-id="11653"></path><path d="M435.44381 691.44381H204.8c-14.140952 0-25.35619-11.215238-25.35619-25.84381 0-14.140952 11.215238-25.35619 25.35619-25.35619h230.64381c14.140952 0 25.35619 11.215238 25.35619 25.35619 0 14.140952-11.215238 25.84381-25.35619 25.84381z" p-id="11654"></path><path d="M435.44381 845.04381H204.8c-14.140952 0-25.35619-11.215238-25.35619-25.84381 0-14.140952 11.215238-25.84381 25.35619-25.84381h230.64381c14.140952 0 25.35619 11.215238 25.35619 25.84381 0 14.140952-11.215238 25.84381-25.35619 25.84381z" p-id="11655"></path><path d="M768 691.44381h-102.4c-14.140952 0-25.35619-11.215238-25.35619-25.84381 0-14.140952 11.215238-25.35619 25.35619-25.35619h102.4c14.140952 0 25.35619 11.215238 25.35619 25.35619 0 14.140952-11.215238 25.84381-25.35619 25.84381z" p-id="11656"></path><path d="M768 845.04381h-102.4c-14.140952 0-25.35619-11.215238-25.35619-25.84381 0-14.140952 11.215238-25.84381 25.35619-25.84381h102.4c14.140952 0 25.35619 11.215238 25.35619 25.84381 0 14.140952-11.215238 25.84381-25.35619 25.84381z" p-id="11657"></path><path d="M950.857143 999.619048H24.380952c-13.653333 0-24.380952-10.727619-24.380952-24.380953s10.727619-24.380952 24.380952-24.380952h926.476191c13.653333 0 24.380952 10.727619 24.380952 24.380952s-10.727619 24.380952-24.380952 24.380953z" p-id="11658"></path></svg>',
|
||||
"menu_type": "api",
|
||||
"disable": True,
|
||||
"url": "/service_pipeline_modelview/api/"
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@ -621,24 +688,24 @@ class Myapp(BaseMyappView):
|
||||
return self.render_template('external_link.html', data=data)
|
||||
|
||||
|
||||
@expose('/schedule/node/<ip>')
|
||||
def schedule_node(self,ip):
|
||||
all_node_json = resource_used['data']
|
||||
for cluster_name in all_node_json:
|
||||
nodes = all_node_json[cluster_name]
|
||||
if ip in nodes:
|
||||
clusters = conf.get('CLUSTERS', {})
|
||||
cluster = clusters[cluster_name]
|
||||
k8s_client = K8s(cluster.get('KUBECONFIG',''))
|
||||
# 获取最新的节点信息
|
||||
nodes = k8s_client.get_node(ip=ip)
|
||||
if nodes:
|
||||
node = nodes[0]
|
||||
enable_train = node['labels'].get('train','true')
|
||||
k8s_client.label_node([ip],{"train":"false" if enable_train=='true' else "true"})
|
||||
break
|
||||
|
||||
return redirect('/myapp/home')
|
||||
# @expose('/schedule/node/<ip>')
|
||||
# def schedule_node(self,ip):
|
||||
# all_node_json = resource_used['data']
|
||||
# for cluster_name in all_node_json:
|
||||
# nodes = all_node_json[cluster_name]
|
||||
# if ip in nodes:
|
||||
# clusters = conf.get('CLUSTERS', {})
|
||||
# cluster = clusters[cluster_name]
|
||||
# k8s_client = K8s(cluster.get('KUBECONFIG',''))
|
||||
# # 获取最新的节点信息
|
||||
# nodes = k8s_client.get_node(ip=ip)
|
||||
# if nodes:
|
||||
# node = nodes[0]
|
||||
# enable_train = node['labels'].get('train','true')
|
||||
# k8s_client.label_node([ip],{"train":"false" if enable_train=='true' else "true"})
|
||||
# break
|
||||
#
|
||||
# return redirect('/myapp/home')
|
||||
|
||||
|
||||
# from myapp import tracer
|
||||
@ -646,7 +713,7 @@ class Myapp(BaseMyappView):
|
||||
|
||||
# 机器学习首页资源弹窗
|
||||
def mlops_traffic(self,url):
|
||||
if 1 or not resource_used['check_time'] or resource_used['check_time'] < (
|
||||
if 1 or not node_resource_used['check_time'] or node_resource_used['check_time'] < (
|
||||
datetime.datetime.now() - datetime.timedelta(minutes=10)):
|
||||
clusters = conf.get('CLUSTERS', {})
|
||||
for cluster_name in clusters:
|
||||
@ -688,10 +755,10 @@ class Myapp(BaseMyappView):
|
||||
all_node_json[node]['used_cpu'] = int(sum(all_node_json[node]['used_cpu']))
|
||||
all_node_json[node]['used_gpu'] = int(sum(all_node_json[node]['used_gpu']))
|
||||
|
||||
resource_used['data'][cluster_name] = all_node_json
|
||||
resource_used['check_time'] = datetime.datetime.now()
|
||||
node_resource_used['data'][cluster_name] = all_node_json
|
||||
node_resource_used['check_time'] = datetime.datetime.now()
|
||||
|
||||
all_node_json = resource_used['data']
|
||||
all_node_json = node_resource_used['data']
|
||||
|
||||
# 数据格式说明 dict:
|
||||
# 'delay': Integer 延时隐藏 单位: 毫秒 0为不隐藏
|
||||
@ -704,7 +771,7 @@ class Myapp(BaseMyappView):
|
||||
message = ''
|
||||
td_html = '<td style="border: 1px solid black;padding: 10px">%s</th>'
|
||||
message += "<tr>%s %s %s %s %s %s %s<tr>" % (
|
||||
td_html % "集群", td_html % "资源组(监控)", td_html % "机器(进出)", td_html % "机型", td_html % "cpu占用率", td_html % "内存占用率",
|
||||
td_html % "集群", td_html % "资源组(监控)", td_html % "机器", td_html % "机型", td_html % "cpu占用率", td_html % "内存占用率",
|
||||
td_html % "gpu占用率")
|
||||
global_cluster_load = {}
|
||||
for cluster_name in all_node_json:
|
||||
@ -746,7 +813,7 @@ class Myapp(BaseMyappView):
|
||||
if enable_train == 'true':
|
||||
ip_html = '<a href="%s">%s</a>' % ("/myapp/schedule/node/%s" % ip, ip)
|
||||
else:
|
||||
ip_html = '<a href="%s"><strike>%s</strike></a>' % ("/myapp/schedule/node/%s" % ip, ip)
|
||||
ip_html = '<a href="%s"><strike>%s</strike></a>' % (cluster_config.get('K8S_DASHBOARD_CLUSTER', '').strip('/')+'/#/node/%s?namespace=default' % ip, ip)
|
||||
else:
|
||||
if enable_train == 'true':
|
||||
ip_html = ip
|
||||
@ -796,14 +863,127 @@ class Myapp(BaseMyappView):
|
||||
# 返回模板
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
|
||||
# pipeline每个任务的资源占用情况
|
||||
def pipeline_task_resource(self,url):
|
||||
if not pipeline_resource_used['check_time'] or pipeline_resource_used['check_time'] < (datetime.datetime.now() - datetime.timedelta(minutes=10)):
|
||||
clusters = conf.get('CLUSTERS', {})
|
||||
all_tasks_json = {}
|
||||
for cluster_name in clusters:
|
||||
cluster = clusters[cluster_name]
|
||||
k8s_client = K8s(cluster.get('KUBECONFIG',''))
|
||||
try:
|
||||
# 获取pod的资源占用
|
||||
all_tasks_json[cluster_name]={}
|
||||
# print(all_node_json)
|
||||
for namespace in ['pipeline', 'katib', 'service']:
|
||||
all_tasks_json[cluster_name][namespace]={}
|
||||
all_pods = k8s_client.get_pods(namespace=namespace)
|
||||
for pod in all_pods:
|
||||
if pod['status'] == 'Running':
|
||||
user = pod['labels'].get('user',pod['labels'].get('username',pod['labels'].get('run-rtx',pod['labels'].get('rtx-user',''))))
|
||||
if user:
|
||||
all_tasks_json[cluster_name][namespace][pod['name']] = {}
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['username'] = user
|
||||
# print(namespace,pod)
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['request_memory']=pod['memory']
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['request_cpu']=pod['cpu']
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['request_gpu']=pod['gpu']
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['namespace']=namespace
|
||||
|
||||
# 获取pod的资源使用
|
||||
all_pods_metrics=k8s_client.get_pod_metrics(namespace=namespace)
|
||||
for pod in all_pods_metrics:
|
||||
if pod['name'] in all_tasks_json[cluster_name][namespace]:
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['used_memory']=pod['memory']
|
||||
all_tasks_json[cluster_name][namespace][pod['name']]['used_cpu']=pod['cpu']
|
||||
except Exception as e:
|
||||
print(e)
|
||||
pipeline_resource_used['data'] = all_tasks_json
|
||||
pipeline_resource_used['check_time'] = datetime.datetime.now()
|
||||
|
||||
# 数据格式说明 dict:
|
||||
# 'delay': Integer 延时隐藏 单位: 毫秒 0为不隐藏
|
||||
# 'hit': Boolean 是否命中
|
||||
# 'target': String 当前目标
|
||||
# 'type': String 类型 目前仅支持html类型
|
||||
# 'title': String 标题
|
||||
# 'content': String 内容html内容
|
||||
# /static/appbuilder/mnt/make_pipeline.mp4
|
||||
all_tasks_json = pipeline_resource_used['data']
|
||||
message = ''
|
||||
td_html = '<td class="ellip1" style="border: 1px solid black;padding: 10px">%s</th>'
|
||||
message += "<tr>%s %s %s %s %s %s %s<tr>" % (
|
||||
td_html % "集群", td_html % "空间",td_html % "容器", td_html % "用户", td_html % "cpu", td_html % "内存",td_html % "gpu")
|
||||
for cluster_name in all_tasks_json:
|
||||
cluster_config = conf.get('CLUSTERS', {}).get(cluster_name, {})
|
||||
for namespace in all_tasks_json[cluster_name]:
|
||||
for pod_name in all_tasks_json[cluster_name][namespace]:
|
||||
pod = all_tasks_json[cluster_name][namespace][pod_name]
|
||||
dashboard_url = cluster_config.get('K8S_DASHBOARD_CLUSTER', '').strip('/') + '/#/search?namespace=%s&q=%s'%(namespace,pod_name)
|
||||
grafana_url = cluster_config.get('GRAFANA_HOST', '').strip('/') + conf.get('GRAFANA_TASK_PATH')
|
||||
message += '<tr>%s %s %s %s %s %s %s<tr>' % (
|
||||
td_html % cluster_name,
|
||||
td_html % ('<a target="blank" href="%s">%s</a>' % (dashboard_url, namespace)),
|
||||
'<td class="ellip1" style="border: 1px solid black;padding: 10px">%s</th>' % ('<a target="blank" href="%s">%s</a>' % (grafana_url+pod_name, pod_name)),
|
||||
td_html % pod['username'],
|
||||
td_html % ("cpu:%s/%s" % (int(int(pod.get('used_cpu','0'))/1000), int(pod.get('request_cpu','0')))),
|
||||
td_html % ("mem:%s/%s" % (int(pod.get('used_memory','0')), int(pod.get('request_memory','0')))),
|
||||
td_html % ("gpu:%s" % (pod.get('request_gpu','')),),
|
||||
)
|
||||
|
||||
|
||||
message = Markup(f'<table>%s</table>' % message)
|
||||
# print(message)
|
||||
data = {
|
||||
'content': message,
|
||||
'delay': 3000,
|
||||
'hit': True,
|
||||
'target': url,
|
||||
'title': '所有用户容器负载',
|
||||
'type': 'html',
|
||||
}
|
||||
# 返回模板
|
||||
return jsonify(data)
|
||||
|
||||
@expose('/feature/check')
|
||||
# @trace(tracer,depth=1,trace_content='line')
|
||||
# @pysnooper.snoop()
|
||||
def featureCheck(self):
|
||||
url = request.values.get("url", type=str, default=None)
|
||||
print(url)
|
||||
if '/myapp/home' in url:
|
||||
return self.mlops_traffic(url)
|
||||
|
||||
if url in '/frontend/train/total_resource':
|
||||
if g.user.username in conf.get('ADMIN_USER',''):
|
||||
return self.pipeline_task_resource(url)
|
||||
|
||||
# pipeline_url = conf.get('MODEL_URLS',{}).get('pipeline','')
|
||||
# if url in pipeline_url:
|
||||
# data = {
|
||||
# 'content': '',
|
||||
# 'delay': 3000,
|
||||
# 'hit': True,
|
||||
# 'target': url,
|
||||
# 'title': '重要通知',
|
||||
# 'type': 'html',
|
||||
# }
|
||||
# # 返回模板
|
||||
# return jsonify(data)
|
||||
|
||||
# data = {
|
||||
# 'content': url,
|
||||
# 'delay': 3000,
|
||||
# 'hit': True,
|
||||
# 'target': url,
|
||||
# 'title': url,
|
||||
# 'type': 'html',
|
||||
# }
|
||||
# return jsonify(data)
|
||||
|
||||
# flash('xxxxxxx','success')
|
||||
return jsonify({})
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user