mirror/cube-studio
2
0
Fork 0
mirror of https://github.com/tencentmusic/cube-studio.git synced 2024-11-21 01:16:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

add sidecar and service monitoring

Browse Source
This commit is contained in:
FerdinandWard 2022-08-19 16:45:59 +08:00
parent 2510707680
commit 28df8a4f19
4 changed files with 31 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
install/kubernetes/create_ns_secret.sh
View File

@ -10,6 +10,6 @@ do
done
kubectl label ns katib katib-metricscollector-injection=enabled --overwrite
kubectl label ns service istio-injection-

18
install/kubernetes/prometheus/grafana/dashboard/istio-service.json
View File

@ -72,7 +72,7 @@
"targets": [
{
"exemplar": true,
"expr": "sum by (destination_service_name) (istio_requests_total{destination_service_namespace=\"$namespace\",destination_service_name=~\"($service)\"})",
"expr": "sum by (destination_workload) (istio_requests_total{destination_service_namespace=\"$namespace\",destination_workload=~\"($service)\"})",
"format": "time_series",
"hide": true,
"interval": "",
@ -82,10 +82,10 @@
},
{
"exemplar": true,
"expr": "sum by (destination_service_name,response_code) (irate(istio_requests_total{destination_service_namespace=\"$namespace\",destination_service_name=~\"($service)\"}[1m]))",
"expr": "sum by (destination_workload,response_code) (irate(istio_requests_total{destination_service_namespace=\"$namespace\",destination_workload=~\"($service)\"}[1m]))",
"hide": false,
"interval": "",
"legendFormat": "{{response_code}} , {{destination_service_name}}",
"legendFormat": "{{response_code}} , {{destination_workload}}",
"refId": "A"
}
],
@ -183,12 +183,12 @@
"targets": [
{
"exemplar": true,
"expr": "sum by (destination_service_name) (irate(istio_request_bytes_sum{destination_service_namespace=\"$namespace\",destination_service_name=~\"($service)\"}[1m]))",
"expr": "sum by (destination_workload) (irate(istio_request_bytes_sum{destination_service_namespace=\"$namespace\",destination_workload=~\"($service)\"}[1m]))",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{destination_service_name}}",
"legendFormat": "{{destination_workload}}",
"refId": "B"
}
],
@ -284,12 +284,12 @@
"targets": [
{
"exemplar": true,
"expr": "avg by (destination_service_name) (rate(istio_request_duration_milliseconds_sum{destination_service_namespace=\"$namespace\",destination_service_name=~\"($service)\"}[1m])/rate(istio_request_duration_milliseconds_count{destination_service_namespace=\"$namespace\",destination_service_name=~\"($service)\"}[1m]))",
"expr": "avg by (destination_workload) (rate(istio_request_duration_milliseconds_sum{destination_service_namespace=\"$namespace\",destination_workload=~\"($service)\"}[1m])/rate(istio_request_duration_milliseconds_count{destination_service_namespace=\"$namespace\",destination_workload=~\"($service)\"}[1m]))",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{destination_service_name}}",
"legendFormat": "{{destination_workload}}",
"refId": "B"
}
],
@ -808,12 +808,12 @@
"targets": [
{
"exemplar": true,
"expr": "sum by (destination_service_name) (istio_requests_total{destination_service_namespace=\"$namespace\"})",
"expr": "sum by (destination_workload) (istio_requests_total{destination_service_namespace=\"$namespace\"})",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{destination_service_name}}",
"legendFormat": "{{destination_workload}}",
"refId": "B"
}
],

19
myapp/utils/py/py_k8s.py
View File

@ -817,12 +817,10 @@ class K8s():
# @pysnooper.snoop()
def make_pod(self,namespace,name,labels,command,args,volume_mount,working_dir,node_selector,resource_memory,resource_cpu,resource_gpu,image_pull_policy,image_pull_secrets,image,hostAliases,env,privileged,accounts,username,ports=None,restart_policy='OnFailure',scheduler_name='default-scheduler',node_name='',health=None):
annotations = None
def make_pod(self,namespace,name,labels,command,args,volume_mount,working_dir,node_selector,resource_memory,resource_cpu,resource_gpu,image_pull_policy,image_pull_secrets,image,hostAliases,env,privileged,accounts,username,ports=None,restart_policy='OnFailure',scheduler_name='default-scheduler',node_name='',health=None,annotations={}):
if scheduler_name == 'kube-batch':
annotations = {
'scheduling.k8s.io/group-name': name
}
annotations['scheduling.k8s.io/group-name']=name
metadata = v1_object_meta.V1ObjectMeta(name=name, namespace=namespace, labels=labels, annotations=annotations)
image_pull_secrets = [client.V1LocalObjectReference(image_pull_secret) for image_pull_secret in image_pull_secrets]
nodeSelector = None
@ -988,12 +986,13 @@ class K8s():
print(e)
# @pysnooper.snoop(watch_explode=())
def create_deployment(self,namespace,name,replicas,labels,command,args,volume_mount,working_dir,node_selector,resource_memory,resource_cpu,resource_gpu,image_pull_policy,image_pull_secrets,image,hostAliases,env,privileged,accounts,username,ports,scheduler_name='default-scheduler',health=None):
def create_deployment(self,namespace,name,replicas,labels,command,args,volume_mount,working_dir,node_selector,resource_memory,resource_cpu,resource_gpu,image_pull_policy,image_pull_secrets,image,hostAliases,env,privileged,accounts,username,ports,scheduler_name='default-scheduler',health=None,annotations={}):
pod,pod_spec = self.make_pod(
namespace=namespace,
name=name,
labels=labels,
annotations=annotations,
command=command,
args=args,
volume_mount=volume_mount,
@ -1086,7 +1085,7 @@ class K8s():
print(e)
# @pysnooper.snoop(watch_explode=())
def create_statefulset(self,namespace,name,replicas,labels,command,args,volume_mount,working_dir,node_selector,resource_memory,resource_cpu,resource_gpu,image_pull_policy,image_pull_secrets,image,hostAliases,env,privileged,accounts,username,ports,restart_policy='Always',scheduler_name='default-scheduler'):
def create_statefulset(self,namespace,name,replicas,labels,command,args,volume_mount,working_dir,node_selector,resource_memory,resource_cpu,resource_gpu,image_pull_policy,image_pull_secrets,image,hostAliases,env,privileged,accounts,username,ports,restart_policy='Always',scheduler_name='default-scheduler',annotations={}):
pod,pod_spec = self.make_pod(
namespace=namespace,
@ -1112,11 +1111,9 @@ class K8s():
restart_policy=restart_policy,
scheduler_name=scheduler_name
)
annotations = None
if scheduler_name == 'kube-batch':
annotations = {
'scheduling.k8s.io/group-name': name
}
annotations['scheduling.k8s.io/group-name'] = name
sts_metadata = v1_object_meta.V1ObjectMeta(name=name, namespace=namespace, labels=labels)
selector = client.models.V1LabelSelector(match_labels=labels)
template_metadata = v1_object_meta.V1ObjectMeta(labels=labels,annotations=annotations)

18
myapp/views/view_inferenceserving.py
View File

@ -90,7 +90,7 @@ class InferenceService_ModelView_base():
# add_columns = ['service_type','project','name', 'label','images','resource_memory','resource_cpu','resource_gpu','min_replicas','max_replicas','ports','host','hpa','metrics','health']
add_columns = ['service_type', 'project', 'label', 'model_name', 'model_version', 'images', 'model_path', 'resource_memory', 'resource_cpu', 'resource_gpu', 'min_replicas', 'max_replicas', 'hpa','priority', 'canary', 'shadow', 'host','inference_config', 'working_dir', 'command','volume_mount', 'env', 'ports', 'metrics', 'health','expand']
add_columns = ['service_type', 'project', 'label', 'model_name', 'model_version', 'images', 'model_path', 'resource_memory', 'resource_cpu', 'resource_gpu', 'min_replicas', 'max_replicas', 'hpa','priority', 'canary', 'shadow', 'host','inference_config', 'working_dir', 'command','volume_mount', 'env', 'ports', 'metrics', 'health','expand','sidecar']
show_columns = ['service_type','project', 'name', 'label','model_name', 'model_version', 'images', 'model_path', 'input_html', 'output_html', 'images', 'volume_mount','working_dir', 'command', 'env', 'resource_memory',
'resource_cpu', 'resource_gpu', 'min_replicas', 'max_replicas', 'ports', 'inference_host_url','hpa','priority', 'canary', 'shadow', 'health','model_status', 'expand_html','metrics_html','deploy_history','host','inference_config']
@ -142,10 +142,12 @@ class InferenceService_ModelView_base():
widget=BS3TextFieldWidget(),validators=[DataRequired()]),
'sidecar': MySelectMultipleField(
_(datamodel.obj.lab('sidecar')), default='',
description='容器的agent代理',
_(datamodel.obj.lab('sidecar')),
default='',
description='容器的agent代理,istio用于服务网格',
widget=Select2ManyWidget(),
choices=[]
validators=[],
choices=[['istio','istio']]
),
"priority": SelectField(
_('服务优先级'),
@ -345,7 +347,7 @@ class InferenceService_ModelView_base():
model_columns = ['service_type', 'project', 'label', 'model_name', 'model_version', 'images', 'model_path']
service_columns = ['resource_memory', 'resource_cpu','resource_gpu', 'min_replicas', 'max_replicas', 'hpa','priority','canary','shadow','host','volume_mount']
service_columns = ['resource_memory', 'resource_cpu','resource_gpu', 'min_replicas', 'max_replicas', 'hpa','priority','canary','shadow','host','volume_mount','sidecar']
admin_columns = ['inference_config','working_dir','command','env','ports','metrics','health','expand']
@ -948,11 +950,17 @@ output %s
pod_ports = list(set(pod_ports))
print('create deployment')
annotations={}
# https://istio.io/latest/docs/reference/config/annotations/
if service.sidecar and 'istio' in service.sidecar and service.service_type=='serving':
labels['sidecar.istio.io/inject']='true'
k8s_client.create_deployment(
namespace=namespace,
name=name,
replicas=deployment_replicas,
labels=labels,
annotations=annotations,
command=['sh','-c',command] if command else None,
args=None,
volume_mount=volume_mount,