remove invalid comments

This commit is contained in:
FerdinandWard 2022-08-08 20:11:53 +08:00
parent e346602378
commit be6f5fb9c2
24 changed files with 110 additions and 169 deletions

View File

@ -12,7 +12,7 @@ from flask import g, request
from jinja2.sandbox import SandboxedEnvironment
from myapp import app
# 模板渲染的上下午函数
# template context
conf = app.config
BASE_CONTEXT = {
"datetime": datetime,

View File

@ -3,7 +3,6 @@ from flask_babel import lazy_gettext as _
import re
from myapp.utils import core
# 定义model
class MyappModelBase():
label_columns={
@ -279,14 +278,12 @@ class MyappModelBase():
return _(re.sub("[._]", " ", col).title())
# 获取node选择器
def get_default_node_selector(self,node_selector,resource_gpu,model_type):
# 先使用项目中定义的选择器
# prefer already defined selectors
if not node_selector:
node_selector=''
# 不使用用户的填写,完全平台决定
# completely determined by the platform
if core.get_gpu(resource_gpu)[0]:
node_selector = node_selector.replace('cpu=true', 'gpu=true') + ",gpu=true,%s=true"%model_type
else:

View File

@ -26,7 +26,6 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -40,7 +39,6 @@ from myapp.utils.py import py_k8s
import pysnooper
# 定义model
class Dataset(Model,AuditMixinNullable,MyappModelBase):
__tablename__ = 'dataset'
id = Column(Integer, primary_key=True)

View File

@ -26,7 +26,6 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -40,10 +39,6 @@ from myapp.utils.py import py_k8s
import pysnooper
# 定义model
class Dimension_table(Model,ImportMixin,MyappModelBase):
__tablename__ = 'dimension'

View File

@ -21,7 +21,6 @@ from myapp.models.helpers import AuditMixinNullable, ImportMixin
from flask import escape, g, Markup, request
from myapp import app,db
from myapp.models.helpers import ImportMixin
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -31,11 +30,10 @@ conf = app.config
# from myapp.utils.py.py_k8s import K8s
# 定义model
class Docker(Model,AuditMixinNullable,MyappModelBase):
__tablename__ = 'docker'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey('project.id'), nullable=False) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'), nullable=False)
project = relationship(
"Project", foreign_keys=[project_id]
)
@ -44,18 +42,16 @@ class Docker(Model,AuditMixinNullable,MyappModelBase):
target_image=Column(String(200), nullable=True,default='')
last_image = Column(String(200), nullable=True, default='')
need_gpu = Column(Boolean, nullable=True, default=False)
consecutive_build = Column(Boolean, default=True) # 连续构建
consecutive_build = Column(Boolean, default=True)
expand = Column(Text(65536), default='{}')
def __repr__(self):
return self.label
# 清空激活
@property
def save(self):
return Markup(f'<a href="/docker_modelview/save/{self.id}">保存</a>')
# 清空激活
@property
def debug(self):
return Markup(f'<a target=_blank href="/docker_modelview/debug/{self.id}">调试</a> | <a href="/docker_modelview/delete_pod/{self.id}">清理</a> | <a target=_blank href="/docker_modelview/save/{self.id}">保存</a>')

View File

@ -25,7 +25,7 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -38,7 +38,7 @@ import re
from myapp.utils.py import py_k8s
import pysnooper
# 定义model
class Repository(Model,AuditMixinNullable,MyappModelBase):
__tablename__ = 'repository'
id = Column(Integer, primary_key=True)
@ -59,18 +59,18 @@ class Repository(Model,AuditMixinNullable,MyappModelBase):
label_columns=MyappModelBase.label_columns.copy()
label_columns.update(label_columns_spec)
# 定义model
class Images(Model,AuditMixinNullable,MyappModelBase):
__tablename__='images'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey('project.id')) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'))
project = relationship(
"Project", foreign_keys=[project_id]
)
name = Column(String(500), nullable=False)
describe = Column(String(1000), nullable=False)
repository_id = Column(Integer, ForeignKey('repository.id')) # 定义外键
repository_id = Column(Integer, ForeignKey('repository.id'))
repository = relationship(
"Repository", foreign_keys=[repository_id]
)
@ -94,17 +94,17 @@ class Images(Model,AuditMixinNullable,MyappModelBase):
return self.name
# 定义model
class Job_Template(Model,AuditMixinNullable,MyappModelBase):
__tablename__='job_template'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey('project.id')) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'))
project = relationship(
"Project", foreign_keys=[project_id]
)
name = Column(String(500), nullable=False,unique=True)
version = Column(Enum('Release','Alpha'),nullable=False,default='Release')
images_id = Column(Integer, ForeignKey('images.id')) # 定义外键
images_id = Column(Integer, ForeignKey('images.id'))
images = relationship(
Images, foreign_keys=[images_id]
)
@ -116,7 +116,7 @@ class Job_Template(Model,AuditMixinNullable,MyappModelBase):
env = Column(Text) # 默认自带的环境变量
volume_mount = Column(String(400),default='') # 强制必须挂载
privileged = Column(Boolean, default=False) # 是否启用特权模式
accounts = Column(String(100)) # 使用账户
accounts = Column(String(100)) # 使用k8s账户
demo=Column(Text)
expand = Column(Text(65536), default='{}')
@ -176,7 +176,6 @@ class Job_Template(Model,AuditMixinNullable,MyappModelBase):
)
# 定义model
class Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
__tablename__ = 'pipeline'
id = Column(Integer, primary_key=True)
@ -195,7 +194,7 @@ class Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
pipeline_argo_id = Column(String(100))
version_id = Column(String(100))
run_id = Column(String(100))
node_selector = Column(String(100), default='cpu=true,train=true') # 挂载
node_selector = Column(String(100), default='cpu=true,train=true')
image_pull_policy = Column(Enum('Always','IfNotPresent'),nullable=False,default='Always')
parallelism = Column(Integer, nullable=False,default=1) # 同一个pipeline最大并行的task数目
alert_status = Column(String(100), default='Pending,Running,Succeeded,Failed,Terminated') # 哪些状态会报警Pending,Running,Succeeded,Failed,Unknown,Waiting,Terminated

View File

@ -26,7 +26,6 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -39,7 +38,7 @@ import re
from myapp.utils.py import py_k8s
import pysnooper
# 定义model
class Metadata_metric(Model,AuditMixinNullable,ImportMixin,MyappModelBase):
__tablename__ = 'metadata_metric'
id = Column(Integer, primary_key=True)

View File

@ -25,7 +25,6 @@ from .model_team import Project
from myapp import app,db
from myapp.models.helpers import ImportMixin
# from myapp.models.base import MyappModel
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -39,13 +38,12 @@ from myapp.utils.py import py_k8s
import pysnooper
# 定义model
class Service_Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
__tablename__ = 'service_pipeline'
id = Column(Integer, primary_key=True)
name = Column(String(100),nullable=False,unique=True)
describe = Column(String(200),nullable=False)
project_id = Column(Integer, ForeignKey('project.id'),nullable=False) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'),nullable=False)
project = relationship(
"Project", foreign_keys=[project_id]
)
@ -53,13 +51,13 @@ class Service_Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase):
namespace=Column(String(100),default='service')
env = Column(String(500),default='')
run_id = Column(String(100))
node_selector = Column(String(100), default='cpu=true,train=true') # 挂载
images = Column(String(200), nullable=False) # 别名
node_selector = Column(String(100), default='cpu=true,train=true')
images = Column(String(200), nullable=False)
working_dir = Column(String(100),default='')
command = Column(String(1000),default='')
volume_mount = Column(String(200),default='') # 挂载
volume_mount = Column(String(200),default='')
image_pull_policy = Column(Enum('Always','IfNotPresent'),nullable=False,default='Always')
replicas = Column(Integer, default=1) # 中控组件的副本数目
replicas = Column(Integer, default=1)
resource_memory = Column(String(100),default='2G')
resource_cpu = Column(String(100), default='2')
resource_gpu= Column(String(100), default='0')

View File

@ -26,7 +26,6 @@ from .model_job import Pipeline
from myapp import app,db
from myapp.models.base import MyappModelBase
from myapp.models.helpers import ImportMixin
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -54,27 +53,27 @@ class service_common():
class Service(Model,AuditMixinNullable,MyappModelBase,service_common):
__tablename__ = 'service'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey('project.id')) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'))
project = relationship(
Project, foreign_keys=[project_id]
)
name = Column(String(100), nullable=False,unique=True) # 用于产生pod service和location
label = Column(String(100), nullable=False) # 别名
images = Column(String(200), nullable=False) # 别名
name = Column(String(100), nullable=False,unique=True) # Used to generate pod service and vs
label = Column(String(100), nullable=False)
images = Column(String(200), nullable=False)
working_dir = Column(String(100),default='')
command = Column(String(1000),default='')
args = Column(Text,default='')
env = Column(Text,default='') # 默认自带的环境变量
volume_mount = Column(String(200),default='') # 挂载
node_selector = Column(String(100),default='cpu=true,serving=true') # 挂载
env = Column(Text,default='')
volume_mount = Column(String(200),default='')
node_selector = Column(String(100),default='cpu=true,serving=true')
replicas = Column(Integer,default=1)
ports = Column(String(100),default='80') # 挂载
ports = Column(String(100),default='80')
resource_memory = Column(String(100),default='2G')
resource_cpu = Column(String(100), default='2')
resource_gpu= Column(String(100), default='0')
deploy_time = Column(String(100), nullable=False,default=datetime.datetime.now)
host = Column(String(200), default='') # 挂载
host = Column(String(200), default='')
expand = Column(Text(65536), default='')
@ -104,16 +103,16 @@ class Service(Model,AuditMixinNullable,MyappModelBase,service_common):
@property
def ip(self):
port = 30000+10*self.id
# 优先使用项目组配置的代理ip
# first, Use the proxy ip configured by the project group
SERVICE_EXTERNAL_IP = json.loads(self.project.expand).get('SERVICE_EXTERNAL_IP',None) if self.project.expand else None
if not SERVICE_EXTERNAL_IP:
# 再使用全局配置代理ip
# second, Use the global configuration proxy ip
SERVICE_EXTERNAL_IP = conf.get('SERVICE_EXTERNAL_IP',[])
if SERVICE_EXTERNAL_IP:
SERVICE_EXTERNAL_IP=SERVICE_EXTERNAL_IP[0]
if not SERVICE_EXTERNAL_IP:
ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # 如果捕获到端口号,要去掉
ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # remove port in host
if core.checkip(ip):
SERVICE_EXTERNAL_IP = ip
@ -138,13 +137,13 @@ class Service(Model,AuditMixinNullable,MyappModelBase,service_common):
class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common):
__tablename__ = 'inferenceservice'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey('project.id')) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'))
project = relationship(
Project, foreign_keys=[project_id]
)
name = Column(String(100), nullable=True,unique=True) # 用于产生pod service和location
label = Column(String(100), nullable=False) # 别名
name = Column(String(100), nullable=True,unique=True)
label = Column(String(100), nullable=False)
service_type= Column(String(100),nullable=True,default='serving')
model_name = Column(String(200),default='')
@ -153,40 +152,40 @@ class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common):
model_type = Column(String(200),default='')
model_input = Column(Text(65536), default='')
model_output = Column(Text(65536), default='')
inference_config = Column(Text(65536), default='') # 推理配置,挂载成configmap
inference_config = Column(Text(65536), default='') # make configmap
model_status = Column(String(200),default='offline')
# model_status = Column(Enum('offline','test','online','delete'),nullable=True,default='offline')
transformer=Column(String(200),default='') # 前后置处理逻辑的文件
transformer=Column(String(200),default='') # pre process and post process
images = Column(String(200), nullable=False) # 别名
images = Column(String(200), nullable=False)
working_dir = Column(String(100),default='')
command = Column(String(1000),default='')
args = Column(Text,default='')
env = Column(Text,default='') # 默认自带的环境变量
volume_mount = Column(String(2000),default='') # 挂载
node_selector = Column(String(100),default='cpu=true,serving=true') # 挂载
env = Column(Text,default='')
volume_mount = Column(String(2000),default='')
node_selector = Column(String(100),default='cpu=true,serving=true')
min_replicas = Column(Integer,default=1)
max_replicas = Column(Integer, default=1)
hpa = Column(String(400), default='') # 弹性伸缩
metrics = Column(Text(65536), default='') # 指标监控
health = Column(String(400), default='') # 健康检查
sidecar = Column(String(400), default='') # sidecar的名称
ports = Column(String(100),default='80') # 挂载
hpa = Column(String(400), default='')
metrics = Column(Text(65536), default='')
health = Column(String(400), default='')
sidecar = Column(String(400), default='')
ports = Column(String(100),default='80')
resource_memory = Column(String(100),default='2G')
resource_cpu = Column(String(100), default='2')
resource_gpu= Column(String(100), default='0')
deploy_time = Column(String(100), nullable=True,default=datetime.datetime.now)
host = Column(String(200), default='') # 挂载
host = Column(String(200), default='')
expand = Column(Text(65536), default='{}')
canary = Column(String(400), default='') # 分流
shadow = Column(String(400), default='') # 镜像
canary = Column(String(400), default='')
shadow = Column(String(400), default='')
run_id = Column(String(100),nullable=True) # 可能同一个pipeline产生多个模型
run_id = Column(String(100),nullable=True)
run_time = Column(String(100))
deploy_history = Column(Text(65536), default='') # 部署记录
deploy_history = Column(Text(65536), default='')
priority = Column(Integer,default=1) # 服务优先级,优先满足高优先级的资源需求
priority = Column(Integer,default=1) # giving priority to meeting high-priority resource needs
@property
def model_name_url(self):
@ -212,9 +211,6 @@ class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common):
<a target=_blank href="{url}">监控</a> |
<a href="/inferenceservice_modelview/clear/{self.id}">清理</a>
'''
# else:
# dom = f'''调试 | 部署测试 | 部署生产 | <a target=_blank href="{url}">监控</a> | 清理'''
# # dom = f'''调试 | 部署测试</a> | 部署生产 | 监控 | 清理 '''
return Markup(dom)
@property
@ -247,16 +243,16 @@ class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common):
def ip(self):
port = 20000+10*self.id
# 优先使用项目组配置的代理ip
# first, Use the proxy ip configured by the project group
SERVICE_EXTERNAL_IP = json.loads(self.project.expand).get('SERVICE_EXTERNAL_IP',None) if self.project.expand else None
if not SERVICE_EXTERNAL_IP:
# 再使用全局配置代理ip
# second, Use the global configuration proxy ip
SERVICE_EXTERNAL_IP = conf.get('SERVICE_EXTERNAL_IP', [])
if SERVICE_EXTERNAL_IP:
SERVICE_EXTERNAL_IP = SERVICE_EXTERNAL_IP[0]
if not SERVICE_EXTERNAL_IP:
ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # 如果捕获到端口号,要去掉
ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # remove port in host
if core.checkip(ip):
SERVICE_EXTERNAL_IP = ip

View File

@ -20,7 +20,7 @@ from myapp.models.helpers import AuditMixinNullable, ImportMixin
from sqlalchemy.orm import backref, relationship
from myapp.models.base import MyappModelBase
from myapp.models.helpers import ImportMixin
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup

View File

@ -23,7 +23,6 @@ from .model_job import Pipeline
from myapp import app,db
from myapp.models.base import MyappModelBase
from myapp.models.helpers import ImportMixin
# 添加自定义model
from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime
from flask_appbuilder.models.decorators import renders
from flask import Markup
@ -32,7 +31,6 @@ metadata = Model.metadata
conf = app.config
# 定义训练 model
class Training_Model(Model,AuditMixinNullable,MyappModelBase):
__tablename__ = 'model'
id = Column(Integer, primary_key=True)
@ -41,12 +39,12 @@ class Training_Model(Model,AuditMixinNullable,MyappModelBase):
describe = Column(String(1000))
path = Column(String(200))
download_url = Column(String(200))
project_id = Column(Integer, ForeignKey('project.id')) # 定义外键
project_id = Column(Integer, ForeignKey('project.id'))
project = relationship(
Project, foreign_keys=[project_id]
)
pipeline_id = Column(Integer,default=0) # 定义外键
run_id = Column(String(100),nullable=False) # 可能同一个pipeline产生多个模型
pipeline_id = Column(Integer,default=0)
run_id = Column(String(100),nullable=False) # pipeline run instance
run_time = Column(String(100))
framework = Column(String(100))
metrics = Column(Text,default='{}')

View File

@ -1,5 +1,4 @@
from myapp import app
# 如果使用 flask run 命令启动,将忽视 这里配置8080而采用默认的5000端口
app.run(host="0.0.0.0", port=80, debug=True)

View File

@ -81,7 +81,7 @@ import json
# 用户列表页面模板
# user list page template
class MyappSecurityListWidget(ListWidget):
"""
Redeclaring to avoid circular imports
@ -89,7 +89,7 @@ class MyappSecurityListWidget(ListWidget):
template = "myapp/fab_overrides/list.html"
# 角色列表页模板
# role list page template
class MyappRoleListWidget(ListWidget):
"""
Role model view from FAB already uses a custom list widget override
@ -102,7 +102,7 @@ class MyappRoleListWidget(ListWidget):
# 自定义list,add,edit页面内容
# customize list,add,edit page
UserModelView.list_columns= ["username", "active", "roles"]
UserModelView.edit_columns= ["first_name", "last_name", "username", "active", "email"]
UserModelView.add_columns= ["first_name", "last_name", "username", "email", "active", "roles"]
@ -115,20 +115,18 @@ PermissionViewModelView.list_widget = MyappSecurityListWidget
PermissionModelView.list_widget = MyappSecurityListWidget
# 自定义扩展系统自带的user
# expand user
from flask_appbuilder.security.sqla.models import User,Role
from sqlalchemy import Column, Integer, ForeignKey, String, Sequence, Table
# 修改绑定
class MyUser(User):
__tablename__ = 'ab_user'
org = Column(String(200)) # 新增的属性,组织架构
org = Column(String(200)) # Organization
def get_full_name(self):
return self.username
# 使用用户名为名称
def __repr__(self):
return self.username
@ -146,10 +144,10 @@ class MyUser(User):
# timestamp = int(func.date_format(self.changed_on))
timestamp = int(self.changed_on.timestamp())
payload = {
"iss": self.username # 用户名作为身份
# "iat": timestamp, # 签发期
# "nbf": timestamp, # 生效期
# "exp": timestamp + 60 * 60 * 24 * 30 * 12, # 有效期12个月
"iss": self.username
# "iat": timestamp, # Issue period
# "nbf": timestamp, # Effective Date
# "exp": timestamp + 60 * 60 * 24 * 30 * 12, # Valid for 12 months
}
global_password = 'myapp'
@ -159,7 +157,7 @@ class MyUser(User):
return ''
# 自定义role view 视图
# customize role view
class MyRoleModelView(RoleModelView):
datamodel = SQLAInterface(Role)
@ -168,7 +166,6 @@ class MyRoleModelView(RoleModelView):
list_columns = ["name", "permissions"]
# 自定义用户展示
class MyUserRemoteUserModelView(UserModelView):
list_columns = ["username", "active", "roles", ]
@ -259,14 +256,14 @@ from myapp.project import Myauthdbview
# @pysnooper.snoop()
class MyappSecurityManager(SecurityManager):
user_model = MyUser # 用户使用自定义的用户
user_model = MyUser
rolemodelview = MyRoleModelView #
# 远程认证
# Remote Authentication
userremoteusermodelview = MyUserRemoteUserModelView
authremoteuserview = MyCustomRemoteUserView
# 账号密码认证
# Account password authentication
userdbmodelview = MyUserRemoteUserModelView
authdbview = Myauthdbview

View File

@ -291,6 +291,7 @@ class BaseMyappView(BaseView):
status=status,
mimetype="application/json",
)
# 前端显示数据
def common_bootstrap_payload(self):
"""Common data always sent to the client"""

View File

@ -47,20 +47,17 @@ class Myapp(BaseMyappView):
else:
msg = 'Hello '+g.user.username+" !"
# 返回模板
return self.render_template('hello.html', msg=msg)
@expose('/home')
def home(self):
# 返回模板
return self.render_template('home.html')
@expose('/menu')
def menu(self):
# 返回模板
menu=[
# 项目组
{
"name": 'group',
"title": '项目空间',

View File

@ -17,7 +17,6 @@ import re,os
from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp import app, appbuilder,db,event_logger
from myapp.utils import core
from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList

View File

@ -11,7 +11,6 @@ import re
from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp.models.model_job import Repository,Images
from myapp.views.view_team import Project_Filter
from myapp import app, appbuilder,db,event_logger
@ -39,7 +38,6 @@ from flask import (
url_for,
)
from myapp import security_manager
import kfp # 使用自定义的就要把pip安装的删除了
from werkzeug.datastructures import FileStorage
from .base import (
api,
@ -76,7 +74,6 @@ class Docker_Filter(MyappFilter):
# 定义数据库视图
class Docker_ModelView_Base():
datamodel = SQLAInterface(Docker)
label_title='docker'
@ -87,7 +84,7 @@ class Docker_ModelView_Base():
conv = GeneralModelConverter(datamodel)
base_permissions = ['can_add', 'can_delete','can_edit', 'can_list', 'can_show'] # 默认为这些
base_order = ('changed_on', 'desc')
base_filters = [["id", Docker_Filter, lambda: []]] # 设置权限过滤器
base_filters = [["id", Docker_Filter, lambda: []]]
order_columns = ['id']
add_columns=['project','describe','base_image','target_image','need_gpu','consecutive_build','expand']
edit_columns=add_columns
@ -265,7 +262,7 @@ class Docker_ModelView_Base():
"loading": True,
"currentHeight": 128
}
# 返回模板
if cluster_name==conf.get('ENVIRONMENT'):
return self.render_template('link.html', data=data)
else:
@ -351,7 +348,6 @@ class Docker_ModelView_Base():
class Docker_ModelView(Docker_ModelView_Base,MyappModelView,DeleteMixin):
datamodel = SQLAInterface(Docker)
# 添加视图和菜单
appbuilder.add_view(Docker_ModelView,"镜像调试",href="/docker_modelview/list/",icon = 'fa-cubes',category = '在线开发',category_icon = 'fa-glass')
# 添加api

View File

@ -10,7 +10,6 @@ import uuid
import re
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp.models.model_job import Repository,Images
from myapp.views.view_team import Project_Filter
from myapp import app, appbuilder,db,event_logger
@ -38,7 +37,6 @@ from flask import (
url_for,
)
from myapp import security_manager
import kfp # 使用自定义的就要把pip安装的删除了
from werkzeug.datastructures import FileStorage
from .base import (
api,
@ -62,13 +60,13 @@ logging = app.logger
# 定义数据库视图
class Repository_ModelView_Base():
datamodel = SQLAInterface(Repository)
label_title='仓库'
check_redirect_list_url = conf.get('MODEL_URLS',{}).get('repository','')
base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些
base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show']
base_order = ('id', 'desc')
order_columns = ['id']
list_columns = ['name','hubsecret','creator','modified']
@ -106,21 +104,21 @@ class Repository_ModelView_Base():
self.add_form_extra_fields['name'] = StringField(
_(self.datamodel.obj.lab('name')),
default=g.user.username+"-",
widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象以及widget函数的参数
widget=BS3TextFieldWidget(),
description = "仓库名称"
)
self.add_form_extra_fields['hubsecret'] = StringField(
_(self.datamodel.obj.lab('hubsecret')),
default=g.user.username + "-hubsecret",
widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象以及widget函数的参数
widget=BS3TextFieldWidget(),
description="在k8s中创建的hub secret",
validators=[Regexp("^[a-z][a-z0-9\-]*[a-z0-9]$"), Length(1, 54),DataRequired()]
)
pre_add_get = set_column
# 直接创建hubsecret
# create hubsecret
# @pysnooper.snoop()
def apply_hubsecret(self,hubsecret):
from myapp.utils.py.py_k8s import K8s
@ -147,10 +145,9 @@ class Repository_ModelView_Base():
# class Repository_ModelView(Repository_ModelView_Base,MyappModelView,DeleteMixin):
# datamodel = SQLAInterface(Repository)
#
# # 添加视图和菜单
# appbuilder.add_view(Repository_ModelView,"仓库",icon = 'fa-shopping-basket',category = '训练',category_icon = 'fa-sitemap')
# 添加api
class Repository_ModelView_Api(Repository_ModelView_Base,MyappModelRestApi):
datamodel = SQLAInterface(Repository)
@ -173,8 +170,6 @@ class Images_Filter(MyappFilter):
return result
# 定义数据库视图
class Images_ModelView_Base():
label_title='镜像'
datamodel = SQLAInterface(Images)
@ -193,22 +188,22 @@ class Images_ModelView_Base():
"dockerfile": StringField(
_(datamodel.obj.lab('dockerfile')),
description='镜像的构建Dockerfile全部内容',
widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象以及widget函数的参数
widget=MyBS3TextAreaFieldWidget(rows=10),
),
"name": StringField(
_(datamodel.obj.lab('name')),
description='镜像名称全称例如ubuntu:20.04',
widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象以及widget函数的参数
widget=BS3TextFieldWidget(),
),
"entrypoint": StringField(
_(datamodel.obj.lab('entrypoint')),
description='镜像的入口命令直接写成单行字符串例如python xx.py无需添加[]',
widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象以及widget函数的参数
widget=BS3TextFieldWidget(),
)
}
edit_form_extra_fields = add_form_extra_fields
base_filters = [["id", Images_Filter, lambda: []]] # 设置权限过滤器
base_filters = [["id", Images_Filter, lambda: []]]
@ -218,7 +213,6 @@ class Images_ModelView(Images_ModelView_Base,MyappModelView,DeleteMixin):
appbuilder.add_view(Images_ModelView,"模板镜像",href="/images_modelview/list/?_flt_2_name=",icon = 'fa-file-image-o',category = '训练')
# 添加api
class Images_ModelView_Api(Images_ModelView_Base,MyappModelRestApi):
datamodel = SQLAInterface(Images)
route_base = '/images_modelview/api'
@ -226,7 +220,3 @@ class Images_ModelView_Api(Images_ModelView_Base,MyappModelRestApi):
appbuilder.add_api(Images_ModelView_Api)
appbuilder.add_separator("训练") # 在指定菜单栏下面的每个子菜单中间添加一个分割线的显示。

View File

@ -17,7 +17,6 @@ import re,os
from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp import app, appbuilder,db,event_logger
from myapp.utils import core
from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList

View File

@ -16,7 +16,6 @@ import re,os
from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp import app, appbuilder,db,event_logger
from myapp.utils import core
from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList
@ -109,7 +108,7 @@ class Metadata_metric_ModelView_base():
add_columns = ['app','metric_data_type','name','label','describe','metric_type','metric_level','metric_dim','metric_responsible','caliber','task_id','public']
# show_columns = ['project','name','describe','config_html','dag_json_html','created_by','changed_by','created_on','changed_on','expand_html']
edit_columns = add_columns
base_filters = [["id", Metadata_Metrics_table_Filter, lambda: []]] # 设置权限过滤器
base_filters = [["id", Metadata_Metrics_table_Filter, lambda: []]]
add_form_extra_fields = {
"app": SelectField(
label=_(datamodel.obj.lab('app')),

View File

@ -10,7 +10,7 @@ import re
import urllib.parse
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob
from myapp.models.model_team import Project,Project_User
from myapp.views.view_team import Project_Join_Filter
@ -65,7 +65,7 @@ from flask import (
)
from myapp import security_manager
from myapp.views.view_team import filter_join_org_project
import kfp # 使用自定义的就要把pip安装的删除了
import kfp
from werkzeug.datastructures import FileStorage
from kubernetes import client as k8s_client
from .base import (
@ -113,7 +113,7 @@ class Pipeline_Filter(MyappFilter):
from sqlalchemy.exc import InvalidRequestError,OperationalError
# 将定义pipeline的流程
# 将dag 转为argo pipeline yaml
# @pysnooper.snoop(watch_explode=())
def dag_to_pipeline(pipeline,dbsession,**kwargs):
if not pipeline.id:

View File

@ -9,7 +9,6 @@ import uuid
import re
from kfp import compiler
from sqlalchemy.exc import InvalidRequestError
# 将model添加成视图并控制在前端的显示
from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob
from myapp.models.model_team import Project,Project_User
from flask_appbuilder.actions import action
@ -57,7 +56,7 @@ from flask import (
url_for,
)
from myapp import security_manager
import kfp # 使用自定义的就要把pip安装的删除了
from werkzeug.datastructures import FileStorage
from .base import (
api,
@ -117,7 +116,7 @@ class Task_ModelView_Base():
add_form_extra_fields = {
"args": StringField(
_(datamodel.obj.lab('args')),
widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象以及widget函数的参数
widget=MyBS3TextAreaFieldWidget(rows=10),
),
"pipeline": QuerySelectField(
datamodel.obj.lab('pipeline'),

View File

@ -4,7 +4,6 @@ from flask_appbuilder import ModelView, ModelRestApi
from flask_appbuilder import ModelView,AppBuilder,expose,BaseView,has_access
from flask_babel import gettext as __
from flask_babel import lazy_gettext as _
# 将model添加成视图并控制在前端的显示
from myapp.models.model_team import Project,Project_User
from flask_appbuilder.actions import action
from wtforms import BooleanField, IntegerField, SelectField, StringField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList
@ -100,7 +99,7 @@ class Project_User_ModelView(Project_User_ModelView_Base,CompactCRUDMixin, Myapp
appbuilder.add_view_no_menu(Project_User_ModelView)
# 添加api
class Project_User_ModelView_Api(Project_User_ModelView_Base,MyappModelRestApi):
datamodel = SQLAInterface(Project_User)
route_base = '/project_user_modelview/api'
@ -135,7 +134,7 @@ class Project_Join_Filter(MyappFilter):
# 获取查询自己所在的项目组的project
# query joined project
def filter_join_org_project():
query = db.session.query(Project)
# user_roles = [role.name.lower() for role in list(get_user_roles())]
@ -143,7 +142,6 @@ def filter_join_org_project():
if g.user.is_admin():
return query.filter(Project.type=='org').order_by(Project.id.desc())
# 查询自己拥有的项目
my_user_id = g.user.get_id() if g.user else 0
owner_ids_query = db.session.query(Project_User.project_id).filter(Project_User.user_id == my_user_id)
@ -153,7 +151,7 @@ def filter_join_org_project():
class Project_ModelView_Base():
label_title='项目组'
datamodel = SQLAInterface(Project)
base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些
base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show']
base_order = ('id', 'desc')
list_columns = ['name','user','type']
cols_width = {
@ -189,9 +187,7 @@ class Project_ModelView_Base():
if not g.user.username in item.get_creators():
raise MyappException('just creator can add/edit')
# 检测是否具有编辑权限只有creator和admin可以编辑
# 打开编辑前,校验权限
# before update, check permission
def pre_update_get(self, item):
self.pre_add_get()
self.check_item_permissions(item)
@ -199,7 +195,7 @@ class Project_ModelView_Base():
flash('just creator can add/edit user','warning')
raise MyappException('just creator can add/edit user')
# 对当前记录是否有权限
# check permission
def check_item_permissions(self,item):
if g.user.is_admin() or g.user.username in item.get_creators():
self.user_permissions = {
@ -218,7 +214,7 @@ class Project_ModelView_Base():
# 添加创始人
# add project user
def post_add(self, item):
if not item.type:
item.type = self.project_type
@ -235,19 +231,18 @@ class Project_ModelView_Base():
# class Project_ModelView_job_template(Project_ModelView_Base,MyappModelView):
# project_type = 'job-template'
# base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器
# base_filters = [["id", Project_Filter, project_type]]
# datamodel = SQLAInterface(Project)
# label_title = '模板分类'
#
# appbuilder.add_view(Project_ModelView_job_template,"模板分类",icon = 'fa-tasks',category = '项目组',category_icon = 'fa-users')
# 添加api
class Project_ModelView_job_template_Api(Project_ModelView_Base,MyappModelRestApi):
route_base = '/project_modelview/job_template/api'
datamodel = SQLAInterface(Project)
project_type = 'job-template'
base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器
base_filters = [["id", Project_Filter, project_type]]
related_views = [Project_User_ModelView_Api, ]
label_title = '模板分类'
edit_form_extra_fields={
@ -271,19 +266,19 @@ appbuilder.add_api(Project_ModelView_job_template_Api)
# class Project_ModelView_org(Project_ModelView_Base,MyappModelView):
# project_type='org'
# base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器
# base_filters = [["id", Project_Filter, project_type]]
# datamodel = SQLAInterface(Project)
# label_title = '项目分组'
#
# appbuilder.add_view(Project_ModelView_org,"项目分组",icon = 'fa-sitemap',category = '项目组',category_icon = 'fa-users')
#
# 添加api
class Project_ModelView_org_Api(Project_ModelView_Base,MyappModelRestApi):
route_base = '/project_modelview/org/api'
datamodel = SQLAInterface(Project)
project_type = 'org'
base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器
base_filters = [["id", Project_Filter, project_type]]
related_views = [Project_User_ModelView_Api, ]
label_title = '项目分组'
edit_form_extra_fields={
@ -309,21 +304,20 @@ appbuilder.add_api(Project_ModelView_org_Api)
# class Project_ModelView_train_model(Project_ModelView_Base,MyappModelView):
# project_type = 'model'
# base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器
# base_filters = [["id", Project_Filter, project_type]]
# datamodel = SQLAInterface(Project)
# label_title = '模型分组'
#
# # 添加视图和菜单
# appbuilder.add_view(Project_ModelView_train_model,"模型分组",icon = 'fa-address-book-o',category = '项目组',category_icon = 'fa-users')
#
# 添加api
class Project_ModelView_train_model_Api(Project_ModelView_Base,MyappModelRestApi):
route_base = '/project_modelview/model/api'
datamodel = SQLAInterface(Project)
project_type = 'model'
label_title = '模型分组'
base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器
base_filters = [["id", Project_Filter, project_type]]
related_views = [Project_User_ModelView_Api, ]
edit_form_extra_fields={
'type': StringField(
@ -340,7 +334,6 @@ appbuilder.add_api(Project_ModelView_train_model_Api)
# 添加api
class Project_ModelView_Api(Project_ModelView_Base,MyappModelRestApi):
datamodel = SQLAInterface(Project)
route_base = '/project_modelview/api'

View File

@ -1,7 +1,6 @@
from flask import render_template,redirect
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask import Blueprint, current_app, jsonify, make_response, request
# 将model添加成视图并控制在前端的显示
from myapp.models.model_serving import Service
from myapp.models.model_train_model import Training_Model
from myapp.models.model_serving import InferenceService
@ -84,12 +83,10 @@ class Training_Model_Filter(MyappFilter):
# 定义数据库视图
# class Training_Model_ModelView(JsonResModelView,DeleteMixin):
class Training_Model_ModelView_Base():
datamodel = SQLAInterface(Training_Model)
base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些
base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show']
base_order = ('changed_on', 'desc')
order_columns = ['id']
list_columns = ['project_url','name','version','framework','api_type','pipeline_url','creator','modified','deploy']
@ -116,7 +113,7 @@ class Training_Model_ModelView_Base():
}
label_title = '模型'
base_filters = [["id", Training_Model_Filter, lambda: []]] # 设置权限过滤器
base_filters = [["id", Training_Model_Filter, lambda: []]]
path_describe= r'''
@ -240,11 +237,10 @@ class Training_Model_ModelView_Base():
class Training_Model_ModelView(Training_Model_ModelView_Base,MyappModelView,DeleteMixin):
datamodel = SQLAInterface(Training_Model)
# 添加视图和菜单
appbuilder.add_view(Training_Model_ModelView,"模型管理",icon = 'fa-hdd-o',category = '服务化',category_icon = 'fa-tasks')
# 添加api
class Training_Model_ModelView_Api(Training_Model_ModelView_Base,MyappModelRestApi): # noqa
datamodel = SQLAInterface(Training_Model)
# base_order = ('id', 'desc')