diff --git a/myapp/models/model_etl_pipeline.py b/myapp/models/model_etl_pipeline.py
index c1daffff..8c5b7311 100644
--- a/myapp/models/model_etl_pipeline.py
+++ b/myapp/models/model_etl_pipeline.py
@@ -25,7 +25,7 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
-# 添加自定义model
+
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@@ -38,7 +38,7 @@ import re
from myapp.utils.py import py_k8s
import pysnooper
-# 定义model
+
class ETL_Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
__tablename__ = 'etl_pipeline'
id = Column(Integer, primary_key=True)
@@ -103,7 +103,7 @@ class ETL_Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
expand=self.expand,
)
-# 定义model
+
class ETL_Task(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
__tablename__ = 'etl_task'
id = Column(Integer, primary_key=True)
diff --git a/myapp/models/model_job.py b/myapp/models/model_job.py
index 6d38d292..88dbec93 100644
--- a/myapp/models/model_job.py
+++ b/myapp/models/model_job.py
@@ -491,7 +491,7 @@ class Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
)
-# 定义model
+
class Task(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
__tablename__ = 'task'
id = Column(Integer, primary_key=True)
@@ -708,7 +708,7 @@ class Crd:
def stop(self):
return Markup(f'停止')
-# 定义model
+
class Workflow(Model,Crd,MyappModelBase):
__tablename__ = 'workflow'
@@ -864,7 +864,7 @@ class Workflow(Model,Crd,MyappModelBase):
def stop(self):
return Markup(f'停止')
-# 定义model
+
class Tfjob(Model,Crd,MyappModelBase):
__tablename__ = 'tfjob'
@@ -894,11 +894,11 @@ class Tfjob(Model,Crd,MyappModelBase):
return Markup(f'未知')
-# 定义model
+
class Xgbjob(Model,Crd,MyappModelBase):
__tablename__ = 'xgbjob'
-# 定义model
+
class Pytorchjob(Model,Crd,MyappModelBase):
__tablename__ = 'pytorchjob'
\ No newline at end of file
diff --git a/myapp/models/model_katib.py b/myapp/models/model_katib.py
deleted file mode 100644
index 619806e2..00000000
--- a/myapp/models/model_katib.py
+++ /dev/null
@@ -1,156 +0,0 @@
-from flask_appbuilder import Model
-from sqlalchemy import Column, Integer, String, ForeignKey,Float
-from sqlalchemy.orm import relationship
-import datetime,time,json
-from sqlalchemy import (
- Boolean,
- Column,
- create_engine,
- DateTime,
- ForeignKey,
- Integer,
- MetaData,
- String,
- Table,
- Text,
- Enum,
-)
-from myapp.utils import core
-import re
-from myapp.models.base import MyappModelBase
-from myapp.models.helpers import AuditMixinNullable, ImportMixin
-from flask import escape, g, Markup, request
-from myapp import app,db
-from myapp.models.helpers import ImportMixin
-# 添加自定义model
-from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
-from flask_appbuilder.models.decorators import renders
-from flask import Markup
-import datetime
-metadata = Model.metadata
-conf = app.config
-
-
-# 定义model
-class Hyperparameter_Tuning(Model,AuditMixinNullable,MyappModelBase):
- __tablename__ = 'hp'
- id = Column(Integer, primary_key=True)
- job_type = Column(Enum('Job','TFJob','XGBJob','PyTorchJob'),nullable=False,default='Job')
- project_id = Column(Integer, ForeignKey('project.id'), nullable=False) # 定义外键
- project = relationship(
- "Project", foreign_keys=[project_id]
- )
- name = Column(String(200), unique = True, nullable=False)
- namespace = Column(String(200), nullable=False,default='katib')
- describe = Column(Text)
- parallel_trial_count = Column(Integer,default=3)
- max_trial_count = Column(Integer,default=12)
- max_failed_trial_count = Column(Integer,default=3)
- objective_type = Column(Enum('maximize','minimize'),nullable=False,default='maximize')
- objective_goal = Column(Float, nullable=False,default=0.99)
- objective_metric_name = Column(String(200), nullable=False,default='Validation-accuracy')
- objective_additional_metric_names = Column(String(200),default='') # 逗号分隔
- algorithm_name = Column(Enum('grid','random','hyperband','bayesianoptimization'),nullable=False,default='random')
- algorithm_setting = Column(Text,default='') # 搜索算法的配置
- parameters=Column(Text,default='{}') # 搜索超参的配置
- job_json = Column(Text, default='{}') # 根据不同算法和参数写入的task模板
- trial_spec=Column(Text,default='') # 根据不同算法和参数写入的task模板
- working_dir = Column(String(200), default='') # 挂载
- volume_mount = Column(String(100), default='') # 挂载
- node_selector = Column(String(100), default='cpu=true,train=true') # 挂载
- image_pull_policy = Column(Enum('Always', 'IfNotPresent'), nullable=False, default='Always')
- resource_memory = Column(String(100), default='1G')
- resource_cpu = Column(String(100), default='1')
-
- experiment=Column(Text,default='') # 构建出来的实验体
- alert_status = Column(String(100), default='') # 哪些状态会报警Pending,Running,Succeeded,Failed,Unknown,Waiting,Terminated
-
-
- def __repr__(self):
- return self.name
-
- @renders('parameters')
- def parameters_html(self):
- return Markup('' + self.parameters + '
')
-
-
-# '''
-# "\"单反斜杠 %5C
-# "|" %7C
-# 回车 %0D%0A
-# 空格 %20
-# 双引号 %22
-# "&" %26
-# '''
- @property
- def name_url(self):
- return Markup(f'{self.name}')
-
- @property
- def describe_url(self):
- return Markup(f'{self.describe}')
-
- @property
- def run_url(self):
- return Markup(f'run')
-
-
-
- @renders('trial_spec')
- def trial_spec_html(self):
- return Markup('' + self.trial_spec + '
')
-
-
- @renders('experiment')
- def experiment_html(self):
- return Markup('' + self.experiment + '
')
-
- def get_node_selector(self):
- return self.get_default_node_selector(self.project.node_selector,self.resource_gpu,'train')
-
-
- def clone(self):
- return Hyperparameter_Tuning(
- name=self.name.replace('_','-'),
- job_type = self.job_type,
- describe=self.describe,
- namespace=self.namespace,
- project_id=self.project_id,
- parallel_trial_count=self.parallel_trial_count,
- max_trial_count=self.max_trial_count,
- max_failed_trial_count=self.max_failed_trial_count,
- objective_type=self.objective_type,
- objective_goal=self.objective_goal,
- objective_metric_name=self.objective_metric_name,
- objective_additional_metric_names=self.objective_additional_metric_names,
- algorithm_name=self.algorithm_name,
- algorithm_setting=self.algorithm_setting,
- parameters=self.parameters,
- job_json = self.job_json,
- trial_spec=self.trial_spec,
- volume_mount=self.volume_mount,
- node_selector=self.node_selector,
- image_pull_policy=self.image_pull_policy,
- resource_memory=self.resource_memory,
- resource_cpu=self.resource_cpu,
- experiment=self.experiment,
- alert_status=self.alert_status
- )
-
-
-
-# 定义model
-from myapp.models.model_job import Crd
-class Experiments(Model,Crd,MyappModelBase):
- __tablename__ = 'experiments'
-
- @property
- def url(self):
- if self.status=='' or self.status=='Created':
- katib_url = conf.get('KATIB_URL') + "/katib/hp_monitor/"
- return Markup(f'{self.name}')
- else:
- katib_url = conf.get('KATIB_URL')+ "/katib/hp_monitor/%s/%s"%(self.namespace,self.name)
- return Markup(f'{self.name}')
-
-
diff --git a/myapp/models/model_metadata.py b/myapp/models/model_metadata.py
index d4da8081..8cd2cc2a 100644
--- a/myapp/models/model_metadata.py
+++ b/myapp/models/model_metadata.py
@@ -26,7 +26,7 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
-# 添加自定义model
+
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@@ -40,7 +40,7 @@ from myapp.utils.py import py_k8s
import pysnooper
-# 定义model
+
class Metadata_table(Model,ImportMixin,MyappModelBase):
__tablename__ = 'metadata_table'
id = Column(Integer, primary_key=True)
diff --git a/myapp/models/model_nni.py b/myapp/models/model_nni.py
index 985907ed..8e8e77d6 100644
--- a/myapp/models/model_nni.py
+++ b/myapp/models/model_nni.py
@@ -22,7 +22,7 @@ from myapp.models.helpers import AuditMixinNullable, ImportMixin
from flask import escape, g, Markup, request
from myapp import app,db
from myapp.models.helpers import ImportMixin
-# 添加自定义model
+
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@@ -31,7 +31,7 @@ metadata = Model.metadata
conf = app.config
-# 定义model
+
class NNI(Model,AuditMixinNullable,MyappModelBase):
__tablename__ = 'nni'
id = Column(Integer, primary_key=True)
diff --git a/myapp/models/model_notebook.py b/myapp/models/model_notebook.py
index a53b9c1b..a7ff328d 100644
--- a/myapp/models/model_notebook.py
+++ b/myapp/models/model_notebook.py
@@ -21,7 +21,7 @@ from myapp.models.helpers import AuditMixinNullable, ImportMixin
from flask import escape, g, Markup, request
from myapp import app,db
from myapp.models.helpers import ImportMixin
-# 添加自定义model
+
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@@ -31,7 +31,7 @@ conf = app.config
from myapp.utils.py import py_k8s
-# 定义model
+
class Notebook(Model,AuditMixinNullable,MyappModelBase):
__tablename__ = 'notebook'
id = Column(Integer, primary_key=True)
diff --git a/myapp/models/model_team.py b/myapp/models/model_team.py
index 8e897ed4..15a358f4 100644
--- a/myapp/models/model_team.py
+++ b/myapp/models/model_team.py
@@ -38,7 +38,7 @@ class Project(Model,AuditMixinNullable,MyappModelBase):
id = Column(Integer, primary_key=True)
name = Column(String(50), nullable=False)
describe = Column(String(500), nullable=False)
- type = Column(String(50)) # 项目类型。组织架构项目组，功能项目组
+ type = Column(String(50)) # org, job_template, model
expand = Column(Text(65536), default='{}')
export_children = ["user"]
diff --git a/myapp/tools/watch_tfjob.py b/myapp/tools/watch_tfjob.py
index 3f21e3ef..61b33cb5 100644
--- a/myapp/tools/watch_tfjob.py
+++ b/myapp/tools/watch_tfjob.py
@@ -41,7 +41,7 @@ else:
print('no kubeconfig in cluster %s' % cluster)
exit(1)
-# 推送微信消息
+# 推送消息
# @pysnooper.snoop()
def deliver_message(tfjob):
if not tfjob:
diff --git a/myapp/utils/py/py_k8s.py b/myapp/utils/py/py_k8s.py
index 8c3aead5..e7e63c84 100755
--- a/myapp/utils/py/py_k8s.py
+++ b/myapp/utils/py/py_k8s.py
@@ -1,7 +1,4 @@
import time,datetime,logging,os,sys
-
-dir_common = os.path.split(os.path.realpath(__file__))[0] + '/../'
-sys.path.append(dir_common) # 将根目录添加到系统目录,才能正常引用common文件夹
import re
from kubernetes import client,config,watch
from kubernetes.client.models import v1_pod,v1_object_meta,v1_pod_spec,v1_deployment,v1_deployment_spec
@@ -19,7 +16,6 @@ from kubernetes import config
from kubernetes.client.rest import ApiException
-# K8s操作类型
class K8s():
def __init__(self,file_path=None): # kubeconfig
@@ -29,7 +25,7 @@ class K8s():
elif kubeconfig:
config.kube_config.load_kube_config(config_file=kubeconfig)
else:
- config.load_incluster_config() # 使用为pod配置的rbac访问集群
+ config.load_incluster_config()
self.v1 = client.CoreV1Api()
self.v1beta1 = client.ExtensionsV1beta1Api()
self.AppsV1Api = client.AppsV1Api()
diff --git a/myapp/views/view_dimension.py b/myapp/views/view_dimension.py
index ff027ef9..1985c9dc 100644
--- a/myapp/views/view_dimension.py
+++ b/myapp/views/view_dimension.py
@@ -22,7 +22,7 @@ import re,os
from sqlalchemy import and_, or_, select
from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp
from kfp import compiler
-# 将model添加成视图，并控制在前端的显示
+
from myapp import app, appbuilder,db,event_logger
from myapp.utils import core
from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList
@@ -209,7 +209,7 @@ class Dimension_table_ModelView_Api(MyappModelRestApi):
"table_html":"表名",
"table_name":"表名"
}
- base_filters = [["id", Dimension_table_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Dimension_table_Filter, lambda: []]]
add_fieldsets = [
(
diff --git a/myapp/views/view_docker.py b/myapp/views/view_docker.py
index d58c31dc..56b92b58 100644
--- a/myapp/views/view_docker.py
+++ b/myapp/views/view_docker.py
@@ -82,7 +82,7 @@ class Docker_ModelView_Base():
crd_name = 'docker'
conv = GeneralModelConverter(datamodel)
- base_permissions = ['can_add', 'can_delete','can_edit', 'can_list', 'can_show'] # 默认为这些
+ base_permissions = ['can_add', 'can_delete','can_edit', 'can_list', 'can_show']
base_order = ('changed_on', 'desc')
base_filters = [["id", Docker_Filter, lambda: []]]
order_columns = ['id']
diff --git a/myapp/views/view_etl_pipeline.py b/myapp/views/view_etl_pipeline.py
index d3c3b473..116adb2e 100644
--- a/myapp/views/view_etl_pipeline.py
+++ b/myapp/views/view_etl_pipeline.py
@@ -10,7 +10,7 @@ import re
import urllib.parse
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_etl_pipeline import ETL_Pipeline,ETL_Task
from myapp.models.model_team import Project,Project_User
from myapp.views.view_team import Project_Join_Filter
@@ -65,7 +65,7 @@ from flask import (
)
from myapp import security_manager
from myapp.views.view_team import filter_join_org_project
-import kfp # 使用自定义的就要把pip安装的删除了
+
from werkzeug.datastructures import FileStorage
from kubernetes import client as k8s_client
from .base import (
@@ -180,7 +180,7 @@ class ETL_Pipeline_ModelView_Base():
edit_columns = ['project','name','describe','created_by']
- base_filters = [["id", ETL_Pipeline_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", ETL_Pipeline_Filter, lambda: []]]
conv = GeneralModelConverter(datamodel)
# related_views = [ETL_Task_ModelView,]
diff --git a/myapp/views/view_inferenceserving.py b/myapp/views/view_inferenceserving.py
index 0d458580..62e5882d 100644
--- a/myapp/views/view_inferenceserving.py
+++ b/myapp/views/view_inferenceserving.py
@@ -1,7 +1,7 @@
from flask import render_template,redirect
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask import Blueprint, current_app, jsonify, make_response, request
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_serving import InferenceService
from myapp.models.model_team import Project,Project_User
from myapp.utils import core
@@ -88,7 +88,7 @@ class InferenceService_ModelView_base():
datamodel = SQLAInterface(InferenceService)
check_redirect_list_url = conf.get('MODEL_URLS',{}).get('inferenceservice','')
- # 外层的add_column和edit_columns 还有show_columns 一定要全，不然在gunicorn形式下get的不一定能被翻译
+
# add_columns = ['service_type','project','name', 'label','images','resource_memory','resource_cpu','resource_gpu','min_replicas','max_replicas','ports','host','hpa','metrics','health']
add_columns = ['service_type', 'project', 'label', 'model_name', 'model_version', 'images', 'model_path', 'resource_memory', 'resource_cpu', 'resource_gpu', 'min_replicas', 'max_replicas', 'hpa','priority', 'canary', 'shadow', 'host','inference_config', 'working_dir', 'command','volume_mount', 'env', 'ports', 'metrics', 'health','expand']
show_columns = ['service_type','project', 'name', 'label','model_name', 'model_version', 'images', 'model_path', 'input_html', 'output_html', 'images', 'volume_mount','working_dir', 'command', 'env', 'resource_memory',
@@ -121,7 +121,7 @@ class InferenceService_ModelView_base():
base_order = ('id','desc')
order_columns = ['id']
- base_filters = [["id",InferenceService_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id",InferenceService_Filter, lambda: []]]
custom_service = 'serving'
service_type_choices= [custom_service,'tfserving','torch-server','onnxruntime','triton-server']
# label_columns = {
diff --git a/myapp/views/view_job_template.py b/myapp/views/view_job_template.py
index b29db6ee..2389d2cd 100644
--- a/myapp/views/view_job_template.py
+++ b/myapp/views/view_job_template.py
@@ -12,7 +12,7 @@ from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regex
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob
from myapp.models.model_team import Project,Project_User
from flask_appbuilder.actions import action
@@ -82,7 +82,7 @@ class Job_Tempalte_Filter(MyappFilter):
# logging.info(join_projects_id)
return query.filter(self.model.version=='Release')
-# 定义数据库视图
+
class Job_Template_ModelView_Base():
datamodel = SQLAInterface(Job_Template)
label_title='任务模板'
@@ -99,7 +99,7 @@ class Job_Template_ModelView_Base():
add_columns = ['project','images','name','version','describe','workdir','entrypoint','volume_mount','job_args_definition','args','env','hostAliases','privileged','accounts','demo','expand']
edit_columns = add_columns
- base_filters = [["id", Job_Tempalte_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Job_Tempalte_Filter, lambda: []]]
base_order = ('id', 'desc')
order_columns = ['id']
add_form_query_rel_fields = {
diff --git a/myapp/views/view_kfserving.py b/myapp/views/view_kfserving.py
deleted file mode 100644
index f8885353..00000000
--- a/myapp/views/view_kfserving.py
+++ /dev/null
@@ -1,372 +0,0 @@
-from flask import render_template,redirect
-from flask_appbuilder.models.sqla.interface import SQLAInterface
-from flask import Blueprint, current_app, jsonify, make_response, request
-# 将model添加成视图，并控制在前端的显示
-from myapp.models.model_serving import Service,KfService
-from myapp.models.model_team import Project,Project_User
-from myapp.utils import core
-from flask_babel import gettext as __
-from flask_babel import lazy_gettext as _
-from flask_appbuilder.actions import action
-from myapp import app, appbuilder,db,event_logger
-import logging
-import re
-import uuid
-import requests
-from myapp.exceptions import MyappException
-from flask_appbuilder.security.decorators import has_access
-from myapp.models.model_job import Repository
-from flask_wtf.file import FileAllowed, FileField, FileRequired
-from werkzeug.datastructures import FileStorage
-from wtforms.ext.sqlalchemy.fields import QuerySelectField
-from myapp import security_manager
-import os,sys
-from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp
-from wtforms import BooleanField, IntegerField, SelectField, StringField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList
-from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget
-from myapp.forms import MyBS3TextAreaFieldWidget,MySelect2Widget,MyCodeArea,MyLineSeparatedListField,MyJSONField,MyBS3TextFieldWidget,MySelectMultipleField
-from myapp.utils.py import py_k8s
-import os, zipfile
-import shutil
-from flask import (
- current_app,
- abort,
- flash,
- g,
- Markup,
- make_response,
- redirect,
- render_template,
- request,
- send_from_directory,
- Response,
- url_for,
-)
-from .base import (
- DeleteMixin,
- api,
- BaseMyappView,
- check_ownership,
- data_payload_response,
- DeleteMixin,
- generate_download_headers,
- get_error_msg,
- get_user_roles,
- handle_api_exception,
- json_error_response,
- json_success,
- MyappFilter,
- MyappModelView,
-
-)
-from sqlalchemy import and_, or_, select
-from .baseApi import (
- MyappModelRestApi
-)
-
-import kubernetes
-from kfserving import KFServingClient
-from kfserving import V1alpha2EndpointSpec
-from kfserving import V1alpha2CustomSpec
-from kfserving import V1alpha2PredictorSpec
-from kfserving import V1alpha2TensorflowSpec
-from kfserving import V1alpha2InferenceServiceSpec
-from kfserving import V1alpha2InferenceService
-
-from flask_appbuilder import CompactCRUDMixin, expose
-import pysnooper,datetime,time,json
-conf = app.config
-
-
-class KfService_ModelView(MyappModelView):
- datamodel = SQLAInterface(KfService)
- crd_name = 'inferenceservice'
- help_url = conf.get('HELP_URL', {}).get(datamodel.obj.__tablename__, '') if datamodel else ''
- show_columns = ['name', 'label','service_type','default_service','canary_service','canary_traffic_percent','k8s_yaml']
- add_columns = ['name', 'label', 'service_type','default_service','canary_service','canary_traffic_percent']
- list_columns = ['label_url','host','service','deploy','status','roll']
- edit_columns = add_columns
- base_order = ('id','desc')
- order_columns = ['id']
-
- @expose('/deploy1/',methods=['POST',"GET"])
- def deploy1(self,kfservice_id):
- mykfservice = db.session.query(KfService).filter_by(id=kfservice_id).first()
- from myapp.utils.py.py_k8s import K8s
- k8s = K8s(mykfservice.project.cluster.get('KUBECONFIG',''))
- namespace = conf.get('KFSERVING_NAMESPACE')
- crd_info = conf.get('CRD_INFO')['inferenceservice']
- crd_list = k8s.get_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'],
- namespace=namespace)
- for crd_obj in crd_list:
- if crd_obj['name'] == mykfservice.name:
- k8s.delete_crd(group=crd_info['group'], version=crd_info['version'], plural=crd_info['plural'],
- namespace=namespace, name=mykfservice.name)
- def get_env(env_str):
- if not env_str:
- return []
- envs = re.split('\r|\n', env_str)
- envs = [env.split('=') for env in envs if env and len(env.split('=')) == 2]
- return envs
-
- def get_kfjson(service,mykfservice):
- if not service:
- return None
-
- image_secrets = conf.get('HUBSECRET', [])
- user_hubsecrets = db.session.query(Repository.hubsecret).filter(Repository.created_by_fk == g.user.id).all()
- if user_hubsecrets:
- for hubsecret in user_hubsecrets:
- if hubsecret[0] not in image_secrets:
- image_secrets.append(hubsecret[0])
-
- kfjson={
- "minReplicas": service.min_replicas,
- "maxReplicas": service.max_replicas,
- "custom": {
- "affinity": {
- "nodeAffinity": {
- "requiredDuringSchedulingIgnoredDuringExecution": {
- "nodeSelectorTerms": [
- {
- "matchExpressions": [
- {
- "key": "gpu" if core.get_gpu(service.resource_gpu)[0] else "cpu",
- "operator": "In",
- "values": [
- "true"
- ]
- },
- ]
- }
- ]
- }
- },
- },
- "imagePullSecrets": [{"name":hubsecret} for hubsecret in image_secrets],
- "container": {
- "image": service.images,
- "imagePullPolicy": conf.get('IMAGE_PULL_POLICY','Always'),
- "name": mykfservice.name+"-"+service.name,
- "workingDir": service.working_dir if service.working_dir else None,
- "command": ["sh", "-c",service.command] if service.command else None,
- "resources": {
- "requests": {
- "cpu": service.resource_cpu,
- "memory": service.resource_memory
- }
- },
- "env":[{"name":env[0],"value":env[1]} for env in get_env(service.env)],
- # "volumeMounts": [
- # {
- # "mountPath": "/mnt/%s" % service.created_by.username,
- # "name": "workspace",
- # "subPath": service.created_by.username
- # }
- # ],
- # "volumeDevices":[
- # {
- # "devicePath": "/data/home/",
- # "name": "workspace"
- # }
- # ]
- }
- # "volumes": [
- # {
- # "name": "workspace",
- # "persistentVolumeClaim": {
- # "claimName": "kubeflow-user-workspace"
- # }
- # }
- # ]
- }
- }
- return kfjson
-
- crd_json={
- "apiVersion": "serving.kubeflow.org/v1alpha2",
- "kind": "InferenceService",
- "metadata": {
- "labels": {
- "app": mykfservice.name
- },
- "name": mykfservice.name,
- "namespace": namespace
- },
- "spec": {
- "canaryTrafficPercent": mykfservice.canary_traffic_percent,
- "default": {
- mykfservice.service_type: get_kfjson(mykfservice.default_service,mykfservice)
- },
- "canary": {
- mykfservice.service_type: get_kfjson(mykfservice.canary_service,mykfservice),
- } if mykfservice.canary_service else None,
-
- }
- }
-
- import yaml
- ya = yaml.load(json.dumps(crd_json))
- ya_str = yaml.safe_dump(ya, default_flow_style=False)
- logging.info(ya_str)
- crd_objects = k8s.create_crd(group=crd_info['group'],version=crd_info['version'],plural=crd_info['plural'],namespace=namespace,body=crd_json)
- flash(category='warning',message='部署启动，一分钟后部署完成')
- return redirect('/kfservice_modelview/list/')
-
- # 创建kfserving
- @expose('/deploy/', methods=['POST', "GET"])
- def deploy(self, kfservice_id):
- mykfservice = db.session.query(KfService).filter_by(id=kfservice_id).first()
-
- namespace = conf.get('KFSERVING_NAMESPACE')
- crd_info = conf.get('CRD_INFO')['inferenceservice']
-
- # 根据service生成container
- def make_container(service,mykfservice):
- from myapp.utils.py.py_k8s import K8s
- k8s = K8s() # 不部署，不需要配置集群信息
- container = k8s.make_container(name=mykfservice.name + "-" + service.name,
- command=["sh", "-c",service.command] if service.command else None,
- args=None,
- volume_mount=None,
- image_pull_policy=conf.get('IMAGE_PULL_POLICY','Always'),
- image=service.images,
- working_dir=service.working_dir if service.working_dir else None,
- env=service.env,
- resource_memory=service.resource_memory,
- resource_cpu = service.resource_cpu,
- resource_gpu= service.resource_gpu,
- username = service.created_by.username
- )
- return container
-
-
- api_version = crd_info['group'] + '/' + crd_info['version']
- default_endpoint_spec = V1alpha2EndpointSpec(
- predictor=V1alpha2PredictorSpec(
- min_replicas= mykfservice.default_service.min_replicas,
- max_replicas=mykfservice.default_service.max_replicas,
- custom=V1alpha2CustomSpec(
- container=make_container(mykfservice.default_service,mykfservice)
- )
- )
- ) if mykfservice.default_service else None
-
- canary_endpoint_spec = V1alpha2EndpointSpec(
- predictor= V1alpha2PredictorSpec(
- min_replicas=mykfservice.canary_service.min_replicas,
- max_replicas=mykfservice.canary_service.max_replicas,
- custom=V1alpha2CustomSpec(
- container=make_container(mykfservice.canary_service,mykfservice)
- )
- )
- ) if mykfservice.canary_service else None
-
- metadata = kubernetes.client.V1ObjectMeta(
- name=mykfservice.name,
- labels={
- "app":mykfservice.name,
- "rtx-user":mykfservice.created_by.username
- },
- namespace=namespace
- )
-
- isvc = V1alpha2InferenceService(
- api_version=api_version,
- kind=crd_info['kind'],
- metadata=metadata,
- spec=V1alpha2InferenceServiceSpec(
- default=default_endpoint_spec,
- canary=canary_endpoint_spec,
- canary_traffic_percent=mykfservice.canary_traffic_percent
- )
- )
-
- KFServing = KFServingClient()
- try:
- KFServing.delete(mykfservice.name, namespace=namespace,version=crd_info['version'])
- except Exception as e:
- print(e)
-
- KFServing.create(isvc,namespace=namespace,version=crd_info['version'])
-
- flash(category='warning', message='部署启动，一分钟后部署完成')
- return redirect('/kfservice_modelview/list/')
-
-
- # 灰度
- @expose('/roll/', methods=['POST', "GET"])
- def roll(self, kfservice_id):
- mykfservice = db.session.query(KfService).filter_by(id=kfservice_id).first()
- namespace = conf.get('KFSERVING_NAMESPACE')
- crd_info = conf.get('CRD_INFO')['inferenceservice']
-
- # 根据service生成container
- def make_container(service, mykfservice):
- from myapp.utils.py.py_k8s import K8s
- k8s = K8s() # 不部署，不需要配置集群信息
- container = k8s.make_container(name=mykfservice.name + "-" + service.name,
- command=["sh", "-c", service.command] if service.command else None,
- args=None,
- volume_mount=None,
- image_pull_policy=conf.get('IMAGE_PULL_POLICY','Always'),
- image=service.images,
- working_dir=service.working_dir if service.working_dir else None,
- env=service.env,
- resource_memory=service.resource_memory,
- resource_cpu=service.resource_cpu,
- resource_gpu=service.resource_gpu,
- username=service.created_by.username,
- ports = service.ports
- )
- return container
-
-
- canary_endpoint_spec = V1alpha2EndpointSpec(
- predictor=V1alpha2PredictorSpec(
- min_replicas=mykfservice.canary_service.min_replicas,
- max_replicas=mykfservice.canary_service.max_replicas,
- custom=V1alpha2CustomSpec(
- container=make_container(mykfservice.canary_service, mykfservice)
- )
- )
- ) if mykfservice.canary_service else None
-
- KFServing = KFServingClient()
- KFServing.rollout_canary(mykfservice.name, canary=canary_endpoint_spec, percent=mykfservice.canary_traffic_percent,
- namespace=namespace, timeout_seconds=120,version=crd_info['version'])
-
- flash(category='warning', message='滚动升级已配置，刷新查看当前流量比例')
- return redirect('/kfservice_modelview/list/')
-
- # 基础批量删除
- # @pysnooper.snoop()
- def base_muldelete(self,items):
- if not items:
- abort(404)
- for item in items:
- try:
- k8s_client = py_k8s.K8s(item.project.cluster.get('KUBECONFIG',''))
- crd_info = conf.get("CRD_INFO", {}).get(self.crd_name, {})
- if crd_info:
- k8s_client.delete_crd(group=crd_info['group'],version=crd_info['version'],plural=crd_info['plural'],namespace=conf.get('KFSERVING_NAMESPACE'),name=item.name)
- except Exception as e:
- flash(str(e), "danger")
-
- def pre_delete(self,item):
- self.base_muldelete([item])
-
- # @event_logger.log_this
- # @expose("/delete/")
- # @has_access
- # def delete(self, pk):
- # pk = self._deserialize_pk_if_composite(pk)
- # self.base_delete(pk)
- # url = url_for(f"{self.endpoint}.list")
- # return redirect(url)
-
-appbuilder.add_view(KfService_ModelView,"kfserving",icon = 'fa-tasks',category = '服务化')
-
-
-
-
diff --git a/myapp/views/view_link.py b/myapp/views/view_link.py
index bddfde39..3adde424 100644
--- a/myapp/views/view_link.py
+++ b/myapp/views/view_link.py
@@ -1,7 +1,7 @@
from flask_babel import gettext as __
from flask_babel import lazy_gettext as _
-# 将model添加成视图，并控制在前端的显示
+
from myapp import app, appbuilder,db,event_logger
from flask import (
diff --git a/myapp/views/view_nni.py b/myapp/views/view_nni.py
index 7c1f1482..acabc4f2 100644
--- a/myapp/views/view_nni.py
+++ b/myapp/views/view_nni.py
@@ -5,7 +5,6 @@ from flask_appbuilder import ModelView,AppBuilder,expose,BaseView,has_access
from importlib import reload
from flask_babel import gettext as __
from flask_babel import lazy_gettext as _
-# 将model添加成视图，并控制在前端的显示
import uuid
from myapp.models.model_nni import NNI
from myapp.models.model_job import Repository
@@ -91,8 +90,6 @@ class NNI_Filter(MyappFilter):
).order_by(self.model.id.desc())
-
-# 定义数据库视图
class NNI_ModelView_Base():
datamodel = SQLAInterface(NNI)
conv = GeneralModelConverter(datamodel)
@@ -100,9 +97,9 @@ class NNI_ModelView_Base():
check_redirect_list_url = conf.get('MODEL_URLS',{}).get('nni','')
- base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些
+ base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show']
base_order = ('id', 'desc')
- base_filters = [["id", NNI_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", NNI_Filter, lambda: []]]
order_columns = ['id']
list_columns = ['project','describe_url','job_type','creator','modified','run','log']
show_columns = ['created_by','changed_by','created_on','changed_on','job_type','name','namespace','describe',
diff --git a/myapp/views/view_notebook.py b/myapp/views/view_notebook.py
index 9c912686..42a05dbf 100644
--- a/myapp/views/view_notebook.py
+++ b/myapp/views/view_notebook.py
@@ -6,7 +6,7 @@ from importlib import reload
from flask_babel import gettext as __
from flask_babel import lazy_gettext as _
import random
-# 将model添加成视图，并控制在前端的显示
+
import uuid
from myapp.models.model_notebook import Notebook
from myapp.models.model_job import Repository
@@ -82,7 +82,7 @@ class Notebook_Filter(MyappFilter):
-# 定义数据库视图
+
class Notebook_ModelView_Base():
datamodel = SQLAInterface(Notebook)
label_title='notebook'
@@ -91,9 +91,9 @@ class Notebook_ModelView_Base():
datamodel = SQLAInterface(Notebook)
conv = GeneralModelConverter(datamodel)
- base_permissions = ['can_add', 'can_delete','can_edit', 'can_list', 'can_show'] # 默认为这些
+ base_permissions = ['can_add', 'can_delete','can_edit', 'can_list', 'can_show']
base_order = ('changed_on', 'desc')
- base_filters = [["id", Notebook_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Notebook_Filter, lambda: []]]
order_columns = ['id']
search_columns = ['created_by']
add_columns = ['project','name','describe','images','working_dir','volume_mount','resource_memory','resource_cpu','resource_gpu']
diff --git a/myapp/views/view_pipeline.py b/myapp/views/view_pipeline.py
index 39f1e842..832bfaac 100644
--- a/myapp/views/view_pipeline.py
+++ b/myapp/views/view_pipeline.py
@@ -633,7 +633,7 @@ class Pipeline_ModelView_Base():
edit_columns = add_columns
- base_filters = [["id", Pipeline_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Pipeline_Filter, lambda: []]]
conv = GeneralModelConverter(datamodel)
diff --git a/myapp/views/view_runhistory.py b/myapp/views/view_runhistory.py
index c78f06ac..ee958047 100644
--- a/myapp/views/view_runhistory.py
+++ b/myapp/views/view_runhistory.py
@@ -1,7 +1,7 @@
from flask import render_template,redirect
from flask_appbuilder.models.sqla.interface import SQLAInterface
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob
from myapp import app, appbuilder,db,event_logger
@@ -68,7 +68,7 @@ class RunHistory_ModelView_Base():
"created_on":{"type": "ellip2", "width": 300}
}
edit_columns = ['status']
- base_filters = [["id", RunHistory_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", RunHistory_Filter, lambda: []]]
add_form_extra_fields = {
"status": SelectField(
_(datamodel.obj.lab('status')),
diff --git a/myapp/views/view_service_pipeline.py b/myapp/views/view_service_pipeline.py
index 9240194f..3a006b74 100644
--- a/myapp/views/view_service_pipeline.py
+++ b/myapp/views/view_service_pipeline.py
@@ -10,7 +10,7 @@ import uuid
import re
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_service_pipeline import Service_Pipeline
from myapp.models.model_job import Repository
from myapp.models.model_team import Project,Project_User
@@ -132,7 +132,7 @@ class Service_Pipeline_ModelView_Base():
edit_columns = add_columns
- base_filters = [["id", Service_Pipeline_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Service_Pipeline_Filter, lambda: []]]
conv = GeneralModelConverter(datamodel)
diff --git a/myapp/views/view_serving.py b/myapp/views/view_serving.py
index 18417c3e..068f675a 100644
--- a/myapp/views/view_serving.py
+++ b/myapp/views/view_serving.py
@@ -1,7 +1,7 @@
from flask import render_template,redirect
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask import Blueprint, current_app, jsonify, make_response, request
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_serving import Service
from myapp.models.model_team import Project,Project_User
from myapp.utils import core
@@ -103,7 +103,7 @@ class Service_ModelView_base():
base_order = ('id','desc')
order_columns = ['id']
label_title = '云原生服务'
- base_filters = [["id", Service_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Service_Filter, lambda: []]]
add_form_query_rel_fields = {
"project": [["name", Project_Join_Filter, 'org']]
}
diff --git a/myapp/views/view_workflow.py b/myapp/views/view_workflow.py
index 0de1c48e..3368363e 100644
--- a/myapp/views/view_workflow.py
+++ b/myapp/views/view_workflow.py
@@ -3,7 +3,7 @@ from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_babel import gettext as __
-# 将model添加成视图，并控制在前端的显示
+
from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob
from myapp.models.model_team import Project,Project_User
from flask_appbuilder.actions import action
@@ -93,7 +93,7 @@ class Crd_ModelView_Base():
# }
crd_name = ''
base_order = ('create_time', 'desc')
- base_filters = [["id", CRD_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", CRD_Filter, lambda: []]]
# list
@@ -216,7 +216,7 @@ class Workflow_Filter(MyappFilter):
# list正在运行的workflow
class Workflow_ModelView_Base(Crd_ModelView_Base):
- base_filters = [["id", Workflow_Filter, lambda: []]] # 设置权限过滤器
+ base_filters = [["id", Workflow_Filter, lambda: []]]
# 删除之前的 workflow和相关容器
# @pysnooper.snoop()