From be6f5fb9c25d0f03259779c09a430cb69d1ad498 Mon Sep 17 00:00:00 2001 From: FerdinandWard Date: Mon, 8 Aug 2022 20:11:53 +0800 Subject: [PATCH] remove invalid comments --- myapp/jinja_context.py | 2 +- myapp/models/base.py | 7 +-- myapp/models/model_dataset.py | 2 - myapp/models/model_dimension.py | 5 -- myapp/models/model_docker.py | 8 +-- myapp/models/model_job.py | 21 ++++---- myapp/models/model_metadata_metric.py | 3 +- myapp/models/model_service_pipeline.py | 12 ++--- myapp/models/model_serving.py | 74 ++++++++++++-------------- myapp/models/model_team.py | 2 +- myapp/models/model_train_model.py | 8 ++- myapp/run.py | 1 - myapp/security.py | 29 +++++----- myapp/views/base.py | 1 + myapp/views/home.py | 5 +- myapp/views/view_dataset.py | 1 - myapp/views/view_docker.py | 8 +-- myapp/views/view_images.py | 30 ++++------- myapp/views/view_metadata.py | 1 - myapp/views/view_metadata_metric.py | 3 +- myapp/views/view_pipeline.py | 6 +-- myapp/views/view_task.py | 5 +- myapp/views/view_team.py | 35 +++++------- myapp/views/view_train_model.py | 10 ++-- 24 files changed, 110 insertions(+), 169 deletions(-) diff --git a/myapp/jinja_context.py b/myapp/jinja_context.py index 88c2e2bd..7f0e8d01 100644 --- a/myapp/jinja_context.py +++ b/myapp/jinja_context.py @@ -12,7 +12,7 @@ from flask import g, request from jinja2.sandbox import SandboxedEnvironment from myapp import app -# 模板渲染的上下午函数 +# template context conf = app.config BASE_CONTEXT = { "datetime": datetime, diff --git a/myapp/models/base.py b/myapp/models/base.py index 44453692..151167d4 100644 --- a/myapp/models/base.py +++ b/myapp/models/base.py @@ -3,7 +3,6 @@ from flask_babel import lazy_gettext as _ import re from myapp.utils import core -# 定义model class MyappModelBase(): label_columns={ @@ -279,14 +278,12 @@ class MyappModelBase(): return _(re.sub("[._]", " ", col).title()) - - # 获取node选择器 def get_default_node_selector(self,node_selector,resource_gpu,model_type): - # 先使用项目中定义的选择器 + # prefer already defined selectors if not node_selector: node_selector='' - # 不使用用户的填写，完全平台决定 + # completely determined by the platform if core.get_gpu(resource_gpu)[0]: node_selector = node_selector.replace('cpu=true', 'gpu=true') + ",gpu=true,%s=true"%model_type else: diff --git a/myapp/models/model_dataset.py b/myapp/models/model_dataset.py index cfc1ee4e..fd5b1d22 100644 --- a/myapp/models/model_dataset.py +++ b/myapp/models/model_dataset.py @@ -26,7 +26,6 @@ from .model_team import Project from myapp import app,db from myapp.models.helpers import ImportMixin # from myapp.models.base import MyappModel -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -40,7 +39,6 @@ from myapp.utils.py import py_k8s import pysnooper -# 定义model class Dataset(Model,AuditMixinNullable,MyappModelBase): __tablename__ = 'dataset' id = Column(Integer, primary_key=True) diff --git a/myapp/models/model_dimension.py b/myapp/models/model_dimension.py index 20bc3461..df92f5cd 100644 --- a/myapp/models/model_dimension.py +++ b/myapp/models/model_dimension.py @@ -26,7 +26,6 @@ from .model_team import Project from myapp import app,db from myapp.models.helpers import ImportMixin # from myapp.models.base import MyappModel -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -40,10 +39,6 @@ from myapp.utils.py import py_k8s import pysnooper - - - -# 定义model class Dimension_table(Model,ImportMixin,MyappModelBase): __tablename__ = 'dimension' diff --git a/myapp/models/model_docker.py b/myapp/models/model_docker.py index bf9217ea..c0798129 100644 --- a/myapp/models/model_docker.py +++ b/myapp/models/model_docker.py @@ -21,7 +21,6 @@ from myapp.models.helpers import AuditMixinNullable, ImportMixin from flask import escape, g, Markup, request from myapp import app,db from myapp.models.helpers import ImportMixin -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -31,11 +30,10 @@ conf = app.config # from myapp.utils.py.py_k8s import K8s -# 定义model class Docker(Model,AuditMixinNullable,MyappModelBase): __tablename__ = 'docker' id = Column(Integer, primary_key=True) - project_id = Column(Integer, ForeignKey('project.id'), nullable=False) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id'), nullable=False) project = relationship( "Project", foreign_keys=[project_id] ) @@ -44,18 +42,16 @@ class Docker(Model,AuditMixinNullable,MyappModelBase): target_image=Column(String(200), nullable=True,default='') last_image = Column(String(200), nullable=True, default='') need_gpu = Column(Boolean, nullable=True, default=False) - consecutive_build = Column(Boolean, default=True) # 连续构建 + consecutive_build = Column(Boolean, default=True) expand = Column(Text(65536), default='{}') def __repr__(self): return self.label - # 清空激活 @property def save(self): return Markup(f'保存') - # 清空激活 @property def debug(self): return Markup(f'调试 | 清理 | 保存') diff --git a/myapp/models/model_job.py b/myapp/models/model_job.py index f9b152b7..6d38d292 100644 --- a/myapp/models/model_job.py +++ b/myapp/models/model_job.py @@ -25,7 +25,7 @@ from .model_team import Project from myapp import app,db from myapp.models.helpers import ImportMixin # from myapp.models.base import MyappModel -# 添加自定义model + from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -38,7 +38,7 @@ import re from myapp.utils.py import py_k8s import pysnooper -# 定义model + class Repository(Model,AuditMixinNullable,MyappModelBase): __tablename__ = 'repository' id = Column(Integer, primary_key=True) @@ -59,18 +59,18 @@ class Repository(Model,AuditMixinNullable,MyappModelBase): label_columns=MyappModelBase.label_columns.copy() label_columns.update(label_columns_spec) -# 定义model + class Images(Model,AuditMixinNullable,MyappModelBase): __tablename__='images' id = Column(Integer, primary_key=True) - project_id = Column(Integer, ForeignKey('project.id')) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id')) project = relationship( "Project", foreign_keys=[project_id] ) name = Column(String(500), nullable=False) describe = Column(String(1000), nullable=False) - repository_id = Column(Integer, ForeignKey('repository.id')) # 定义外键 + repository_id = Column(Integer, ForeignKey('repository.id')) repository = relationship( "Repository", foreign_keys=[repository_id] ) @@ -94,17 +94,17 @@ class Images(Model,AuditMixinNullable,MyappModelBase): return self.name -# 定义model + class Job_Template(Model,AuditMixinNullable,MyappModelBase): __tablename__='job_template' id = Column(Integer, primary_key=True) - project_id = Column(Integer, ForeignKey('project.id')) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id')) project = relationship( "Project", foreign_keys=[project_id] ) name = Column(String(500), nullable=False,unique=True) version = Column(Enum('Release','Alpha'),nullable=False,default='Release') - images_id = Column(Integer, ForeignKey('images.id')) # 定义外键 + images_id = Column(Integer, ForeignKey('images.id')) images = relationship( Images, foreign_keys=[images_id] ) @@ -116,7 +116,7 @@ class Job_Template(Model,AuditMixinNullable,MyappModelBase): env = Column(Text) # 默认自带的环境变量 volume_mount = Column(String(400),default='') # 强制必须挂载 privileged = Column(Boolean, default=False) # 是否启用特权模式 - accounts = Column(String(100)) # 使用账户 + accounts = Column(String(100)) # 使用k8s账户 demo=Column(Text) expand = Column(Text(65536), default='{}') @@ -176,7 +176,6 @@ class Job_Template(Model,AuditMixinNullable,MyappModelBase): ) -# 定义model class Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase): __tablename__ = 'pipeline' id = Column(Integer, primary_key=True) @@ -195,7 +194,7 @@ class Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase): pipeline_argo_id = Column(String(100)) version_id = Column(String(100)) run_id = Column(String(100)) - node_selector = Column(String(100), default='cpu=true,train=true') # 挂载 + node_selector = Column(String(100), default='cpu=true,train=true') image_pull_policy = Column(Enum('Always','IfNotPresent'),nullable=False,default='Always') parallelism = Column(Integer, nullable=False,default=1) # 同一个pipeline，最大并行的task数目 alert_status = Column(String(100), default='Pending,Running,Succeeded,Failed,Terminated') # 哪些状态会报警Pending,Running,Succeeded,Failed,Unknown,Waiting,Terminated diff --git a/myapp/models/model_metadata_metric.py b/myapp/models/model_metadata_metric.py index 73a3f301..cc3a3c90 100644 --- a/myapp/models/model_metadata_metric.py +++ b/myapp/models/model_metadata_metric.py @@ -26,7 +26,6 @@ from .model_team import Project from myapp import app,db from myapp.models.helpers import ImportMixin # from myapp.models.base import MyappModel -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -39,7 +38,7 @@ import re from myapp.utils.py import py_k8s import pysnooper -# 定义model + class Metadata_metric(Model,AuditMixinNullable,ImportMixin,MyappModelBase): __tablename__ = 'metadata_metric' id = Column(Integer, primary_key=True) diff --git a/myapp/models/model_service_pipeline.py b/myapp/models/model_service_pipeline.py index 1b934181..c26f3d94 100644 --- a/myapp/models/model_service_pipeline.py +++ b/myapp/models/model_service_pipeline.py @@ -25,7 +25,6 @@ from .model_team import Project from myapp import app,db from myapp.models.helpers import ImportMixin # from myapp.models.base import MyappModel -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -39,13 +38,12 @@ from myapp.utils.py import py_k8s import pysnooper -# 定义model class Service_Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase): __tablename__ = 'service_pipeline' id = Column(Integer, primary_key=True) name = Column(String(100),nullable=False,unique=True) describe = Column(String(200),nullable=False) - project_id = Column(Integer, ForeignKey('project.id'),nullable=False) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id'),nullable=False) project = relationship( "Project", foreign_keys=[project_id] ) @@ -53,13 +51,13 @@ class Service_Pipeline(Model,ImportMixin,AuditMixinNullable,MyappModelBase): namespace=Column(String(100),default='service') env = Column(String(500),default='') run_id = Column(String(100)) - node_selector = Column(String(100), default='cpu=true,train=true') # 挂载 - images = Column(String(200), nullable=False) # 别名 + node_selector = Column(String(100), default='cpu=true,train=true') + images = Column(String(200), nullable=False) working_dir = Column(String(100),default='') command = Column(String(1000),default='') - volume_mount = Column(String(200),default='') # 挂载 + volume_mount = Column(String(200),default='') image_pull_policy = Column(Enum('Always','IfNotPresent'),nullable=False,default='Always') - replicas = Column(Integer, default=1) # 中控组件的副本数目 + replicas = Column(Integer, default=1) resource_memory = Column(String(100),default='2G') resource_cpu = Column(String(100), default='2') resource_gpu= Column(String(100), default='0') diff --git a/myapp/models/model_serving.py b/myapp/models/model_serving.py index 3c9ae9de..de7c2e7c 100644 --- a/myapp/models/model_serving.py +++ b/myapp/models/model_serving.py @@ -26,7 +26,6 @@ from .model_job import Pipeline from myapp import app,db from myapp.models.base import MyappModelBase from myapp.models.helpers import ImportMixin -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -54,27 +53,27 @@ class service_common(): class Service(Model,AuditMixinNullable,MyappModelBase,service_common): __tablename__ = 'service' id = Column(Integer, primary_key=True) - project_id = Column(Integer, ForeignKey('project.id')) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id')) project = relationship( Project, foreign_keys=[project_id] ) - name = Column(String(100), nullable=False,unique=True) # 用于产生pod service和location - label = Column(String(100), nullable=False) # 别名 - images = Column(String(200), nullable=False) # 别名 + name = Column(String(100), nullable=False,unique=True) # Used to generate pod service and vs + label = Column(String(100), nullable=False) + images = Column(String(200), nullable=False) working_dir = Column(String(100),default='') command = Column(String(1000),default='') args = Column(Text,default='') - env = Column(Text,default='') # 默认自带的环境变量 - volume_mount = Column(String(200),default='') # 挂载 - node_selector = Column(String(100),default='cpu=true,serving=true') # 挂载 + env = Column(Text,default='') + volume_mount = Column(String(200),default='') + node_selector = Column(String(100),default='cpu=true,serving=true') replicas = Column(Integer,default=1) - ports = Column(String(100),default='80') # 挂载 + ports = Column(String(100),default='80') resource_memory = Column(String(100),default='2G') resource_cpu = Column(String(100), default='2') resource_gpu= Column(String(100), default='0') deploy_time = Column(String(100), nullable=False,default=datetime.datetime.now) - host = Column(String(200), default='') # 挂载 + host = Column(String(200), default='') expand = Column(Text(65536), default='') @@ -104,16 +103,16 @@ class Service(Model,AuditMixinNullable,MyappModelBase,service_common): @property def ip(self): port = 30000+10*self.id - # 优先使用项目组配置的代理ip + # first, Use the proxy ip configured by the project group SERVICE_EXTERNAL_IP = json.loads(self.project.expand).get('SERVICE_EXTERNAL_IP',None) if self.project.expand else None if not SERVICE_EXTERNAL_IP: - # 再使用全局配置代理ip + # second, Use the global configuration proxy ip SERVICE_EXTERNAL_IP = conf.get('SERVICE_EXTERNAL_IP',[]) if SERVICE_EXTERNAL_IP: SERVICE_EXTERNAL_IP=SERVICE_EXTERNAL_IP[0] if not SERVICE_EXTERNAL_IP: - ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # 如果捕获到端口号，要去掉 + ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # remove port in host if core.checkip(ip): SERVICE_EXTERNAL_IP = ip @@ -138,13 +137,13 @@ class Service(Model,AuditMixinNullable,MyappModelBase,service_common): class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common): __tablename__ = 'inferenceservice' id = Column(Integer, primary_key=True) - project_id = Column(Integer, ForeignKey('project.id')) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id')) project = relationship( Project, foreign_keys=[project_id] ) - name = Column(String(100), nullable=True,unique=True) # 用于产生pod service和location - label = Column(String(100), nullable=False) # 别名 + name = Column(String(100), nullable=True,unique=True) + label = Column(String(100), nullable=False) service_type= Column(String(100),nullable=True,default='serving') model_name = Column(String(200),default='') @@ -153,40 +152,40 @@ class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common): model_type = Column(String(200),default='') model_input = Column(Text(65536), default='') model_output = Column(Text(65536), default='') - inference_config = Column(Text(65536), default='') # 推理配置，挂载成configmap + inference_config = Column(Text(65536), default='') # make configmap model_status = Column(String(200),default='offline') # model_status = Column(Enum('offline','test','online','delete'),nullable=True,default='offline') - transformer=Column(String(200),default='') # 前后置处理逻辑的文件 + transformer=Column(String(200),default='') # pre process and post process - images = Column(String(200), nullable=False) # 别名 + images = Column(String(200), nullable=False) working_dir = Column(String(100),default='') command = Column(String(1000),default='') args = Column(Text,default='') - env = Column(Text,default='') # 默认自带的环境变量 - volume_mount = Column(String(2000),default='') # 挂载 - node_selector = Column(String(100),default='cpu=true,serving=true') # 挂载 + env = Column(Text,default='') + volume_mount = Column(String(2000),default='') + node_selector = Column(String(100),default='cpu=true,serving=true') min_replicas = Column(Integer,default=1) max_replicas = Column(Integer, default=1) - hpa = Column(String(400), default='') # 弹性伸缩 - metrics = Column(Text(65536), default='') # 指标监控 - health = Column(String(400), default='') # 健康检查 - sidecar = Column(String(400), default='') # sidecar的名称 - ports = Column(String(100),default='80') # 挂载 + hpa = Column(String(400), default='') + metrics = Column(Text(65536), default='') + health = Column(String(400), default='') + sidecar = Column(String(400), default='') + ports = Column(String(100),default='80') resource_memory = Column(String(100),default='2G') resource_cpu = Column(String(100), default='2') resource_gpu= Column(String(100), default='0') deploy_time = Column(String(100), nullable=True,default=datetime.datetime.now) - host = Column(String(200), default='') # 挂载 + host = Column(String(200), default='') expand = Column(Text(65536), default='{}') - canary = Column(String(400), default='') # 分流 - shadow = Column(String(400), default='') # 镜像 + canary = Column(String(400), default='') + shadow = Column(String(400), default='') - run_id = Column(String(100),nullable=True) # 可能同一个pipeline产生多个模型 + run_id = Column(String(100),nullable=True) run_time = Column(String(100)) - deploy_history = Column(Text(65536), default='') # 部署记录 + deploy_history = Column(Text(65536), default='') - priority = Column(Integer,default=1) # 服务优先级，优先满足高优先级的资源需求 + priority = Column(Integer,default=1) # giving priority to meeting high-priority resource needs @property def model_name_url(self): @@ -212,9 +211,6 @@ class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common): 监控 | 清理 ''' - # else: - # dom = f'''调试 | 部署测试 | 部署生产 | 监控 | 清理''' - # # dom = f'''调试 | 部署测试 | 部署生产 | 监控 | 清理 ''' return Markup(dom) @property @@ -247,16 +243,16 @@ class InferenceService(Model,AuditMixinNullable,MyappModelBase,service_common): def ip(self): port = 20000+10*self.id - # 优先使用项目组配置的代理ip + # first, Use the proxy ip configured by the project group SERVICE_EXTERNAL_IP = json.loads(self.project.expand).get('SERVICE_EXTERNAL_IP',None) if self.project.expand else None if not SERVICE_EXTERNAL_IP: - # 再使用全局配置代理ip + # second, Use the global configuration proxy ip SERVICE_EXTERNAL_IP = conf.get('SERVICE_EXTERNAL_IP', []) if SERVICE_EXTERNAL_IP: SERVICE_EXTERNAL_IP = SERVICE_EXTERNAL_IP[0] if not SERVICE_EXTERNAL_IP: - ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # 如果捕获到端口号，要去掉 + ip = request.host[:request.host.rindex(':')] if ':' in request.host else request.host # remove port in host if core.checkip(ip): SERVICE_EXTERNAL_IP = ip diff --git a/myapp/models/model_team.py b/myapp/models/model_team.py index 9b075215..8e897ed4 100644 --- a/myapp/models/model_team.py +++ b/myapp/models/model_team.py @@ -20,7 +20,7 @@ from myapp.models.helpers import AuditMixinNullable, ImportMixin from sqlalchemy.orm import backref, relationship from myapp.models.base import MyappModelBase from myapp.models.helpers import ImportMixin -# 添加自定义model + from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup diff --git a/myapp/models/model_train_model.py b/myapp/models/model_train_model.py index 68fd6583..638c9e18 100644 --- a/myapp/models/model_train_model.py +++ b/myapp/models/model_train_model.py @@ -23,7 +23,6 @@ from .model_job import Pipeline from myapp import app,db from myapp.models.base import MyappModelBase from myapp.models.helpers import ImportMixin -# 添加自定义model from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime from flask_appbuilder.models.decorators import renders from flask import Markup @@ -32,7 +31,6 @@ metadata = Model.metadata conf = app.config -# 定义训练 model class Training_Model(Model,AuditMixinNullable,MyappModelBase): __tablename__ = 'model' id = Column(Integer, primary_key=True) @@ -41,12 +39,12 @@ class Training_Model(Model,AuditMixinNullable,MyappModelBase): describe = Column(String(1000)) path = Column(String(200)) download_url = Column(String(200)) - project_id = Column(Integer, ForeignKey('project.id')) # 定义外键 + project_id = Column(Integer, ForeignKey('project.id')) project = relationship( Project, foreign_keys=[project_id] ) - pipeline_id = Column(Integer,default=0) # 定义外键 - run_id = Column(String(100),nullable=False) # 可能同一个pipeline产生多个模型 + pipeline_id = Column(Integer,default=0) + run_id = Column(String(100),nullable=False) # pipeline run instance run_time = Column(String(100)) framework = Column(String(100)) metrics = Column(Text,default='{}') diff --git a/myapp/run.py b/myapp/run.py index 841c86a9..ebcd81a4 100644 --- a/myapp/run.py +++ b/myapp/run.py @@ -1,5 +1,4 @@ from myapp import app -# 如果使用 flask run 命令启动，将忽视 这里配置8080，而采用默认的5000端口 app.run(host="0.0.0.0", port=80, debug=True) diff --git a/myapp/security.py b/myapp/security.py index 778538bb..f2de2cad 100644 --- a/myapp/security.py +++ b/myapp/security.py @@ -81,7 +81,7 @@ import json -# 用户列表页面模板 +# user list page template class MyappSecurityListWidget(ListWidget): """ Redeclaring to avoid circular imports @@ -89,7 +89,7 @@ class MyappSecurityListWidget(ListWidget): template = "myapp/fab_overrides/list.html" -# 角色列表页模板 +# role list page template class MyappRoleListWidget(ListWidget): """ Role model view from FAB already uses a custom list widget override @@ -102,7 +102,7 @@ class MyappRoleListWidget(ListWidget): -# 自定义list,add,edit页面内容 +# customize list,add,edit page UserModelView.list_columns= ["username", "active", "roles"] UserModelView.edit_columns= ["first_name", "last_name", "username", "active", "email"] UserModelView.add_columns= ["first_name", "last_name", "username", "email", "active", "roles"] @@ -115,20 +115,18 @@ PermissionViewModelView.list_widget = MyappSecurityListWidget PermissionModelView.list_widget = MyappSecurityListWidget -# 自定义扩展系统自带的user +# expand user from flask_appbuilder.security.sqla.models import User,Role from sqlalchemy import Column, Integer, ForeignKey, String, Sequence, Table -# 修改绑定 class MyUser(User): __tablename__ = 'ab_user' - org = Column(String(200)) # 新增的属性，组织架构 + org = Column(String(200)) # Organization def get_full_name(self): return self.username - # 使用用户名为名称 def __repr__(self): return self.username @@ -146,10 +144,10 @@ class MyUser(User): # timestamp = int(func.date_format(self.changed_on)) timestamp = int(self.changed_on.timestamp()) payload = { - "iss": self.username # 用户名作为身份 - # "iat": timestamp, # 签发期 - # "nbf": timestamp, # 生效期 - # "exp": timestamp + 60 * 60 * 24 * 30 * 12, # 有效期12个月 + "iss": self.username + # "iat": timestamp, # Issue period + # "nbf": timestamp, # Effective Date + # "exp": timestamp + 60 * 60 * 24 * 30 * 12, # Valid for 12 months } global_password = 'myapp' @@ -159,7 +157,7 @@ class MyUser(User): return '' -# 自定义role view 视图 +# customize role view class MyRoleModelView(RoleModelView): datamodel = SQLAInterface(Role) @@ -168,7 +166,6 @@ class MyRoleModelView(RoleModelView): list_columns = ["name", "permissions"] -# 自定义用户展示 class MyUserRemoteUserModelView(UserModelView): list_columns = ["username", "active", "roles", ] @@ -259,14 +256,14 @@ from myapp.project import Myauthdbview # @pysnooper.snoop() class MyappSecurityManager(SecurityManager): - user_model = MyUser # 用户使用自定义的用户 + user_model = MyUser rolemodelview = MyRoleModelView # - # 远程认证 + # Remote Authentication userremoteusermodelview = MyUserRemoteUserModelView authremoteuserview = MyCustomRemoteUserView - # 账号密码认证 + # Account password authentication userdbmodelview = MyUserRemoteUserModelView authdbview = Myauthdbview diff --git a/myapp/views/base.py b/myapp/views/base.py index 585638c5..5b3315c4 100644 --- a/myapp/views/base.py +++ b/myapp/views/base.py @@ -291,6 +291,7 @@ class BaseMyappView(BaseView): status=status, mimetype="application/json", ) + # 前端显示数据 def common_bootstrap_payload(self): """Common data always sent to the client""" diff --git a/myapp/views/home.py b/myapp/views/home.py index e5130f1f..603700f1 100644 --- a/myapp/views/home.py +++ b/myapp/views/home.py @@ -47,20 +47,17 @@ class Myapp(BaseMyappView): else: msg = 'Hello '+g.user.username+" !" - # 返回模板 return self.render_template('hello.html', msg=msg) @expose('/home') def home(self): - # 返回模板 return self.render_template('home.html') @expose('/menu') def menu(self): - # 返回模板 + menu=[ - # 项目组 { "name": 'group', "title": '项目空间', diff --git a/myapp/views/view_dataset.py b/myapp/views/view_dataset.py index c2afd8c9..fc8eebee 100644 --- a/myapp/views/view_dataset.py +++ b/myapp/views/view_dataset.py @@ -17,7 +17,6 @@ import re,os from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 from myapp import app, appbuilder,db,event_logger from myapp.utils import core from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList diff --git a/myapp/views/view_docker.py b/myapp/views/view_docker.py index e4f1e2b9..d58c31dc 100644 --- a/myapp/views/view_docker.py +++ b/myapp/views/view_docker.py @@ -11,7 +11,6 @@ import re from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 from myapp.models.model_job import Repository,Images from myapp.views.view_team import Project_Filter from myapp import app, appbuilder,db,event_logger @@ -39,7 +38,6 @@ from flask import ( url_for, ) from myapp import security_manager -import kfp # 使用自定义的就要把pip安装的删除了 from werkzeug.datastructures import FileStorage from .base import ( api, @@ -76,7 +74,6 @@ class Docker_Filter(MyappFilter): -# 定义数据库视图 class Docker_ModelView_Base(): datamodel = SQLAInterface(Docker) label_title='docker' @@ -87,7 +84,7 @@ class Docker_ModelView_Base(): conv = GeneralModelConverter(datamodel) base_permissions = ['can_add', 'can_delete','can_edit', 'can_list', 'can_show'] # 默认为这些 base_order = ('changed_on', 'desc') - base_filters = [["id", Docker_Filter, lambda: []]] # 设置权限过滤器 + base_filters = [["id", Docker_Filter, lambda: []]] order_columns = ['id'] add_columns=['project','describe','base_image','target_image','need_gpu','consecutive_build','expand'] edit_columns=add_columns @@ -265,7 +262,7 @@ class Docker_ModelView_Base(): "loading": True, "currentHeight": 128 } - # 返回模板 + if cluster_name==conf.get('ENVIRONMENT'): return self.render_template('link.html', data=data) else: @@ -351,7 +348,6 @@ class Docker_ModelView_Base(): class Docker_ModelView(Docker_ModelView_Base,MyappModelView,DeleteMixin): datamodel = SQLAInterface(Docker) -# 添加视图和菜单 appbuilder.add_view(Docker_ModelView,"镜像调试",href="/docker_modelview/list/",icon = 'fa-cubes',category = '在线开发',category_icon = 'fa-glass') # 添加api diff --git a/myapp/views/view_images.py b/myapp/views/view_images.py index e8402730..5c26f57b 100644 --- a/myapp/views/view_images.py +++ b/myapp/views/view_images.py @@ -10,7 +10,6 @@ import uuid import re from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 from myapp.models.model_job import Repository,Images from myapp.views.view_team import Project_Filter from myapp import app, appbuilder,db,event_logger @@ -38,7 +37,6 @@ from flask import ( url_for, ) from myapp import security_manager -import kfp # 使用自定义的就要把pip安装的删除了 from werkzeug.datastructures import FileStorage from .base import ( api, @@ -62,13 +60,13 @@ logging = app.logger -# 定义数据库视图 + class Repository_ModelView_Base(): datamodel = SQLAInterface(Repository) label_title='仓库' check_redirect_list_url = conf.get('MODEL_URLS',{}).get('repository','') - base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些 + base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] base_order = ('id', 'desc') order_columns = ['id'] list_columns = ['name','hubsecret','creator','modified'] @@ -106,21 +104,21 @@ class Repository_ModelView_Base(): self.add_form_extra_fields['name'] = StringField( _(self.datamodel.obj.lab('name')), default=g.user.username+"-", - widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象，以及widget函数的参数 + widget=BS3TextFieldWidget(), description = "仓库名称" ) self.add_form_extra_fields['hubsecret'] = StringField( _(self.datamodel.obj.lab('hubsecret')), default=g.user.username + "-hubsecret", - widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象，以及widget函数的参数 + widget=BS3TextFieldWidget(), description="在k8s中创建的hub secret", validators=[Regexp("^[a-z][a-z0-9\-]*[a-z0-9]$"), Length(1, 54),DataRequired()] ) pre_add_get = set_column - # 直接创建hubsecret + # create hubsecret # @pysnooper.snoop() def apply_hubsecret(self,hubsecret): from myapp.utils.py.py_k8s import K8s @@ -147,10 +145,9 @@ class Repository_ModelView_Base(): # class Repository_ModelView(Repository_ModelView_Base,MyappModelView,DeleteMixin): # datamodel = SQLAInterface(Repository) # -# # 添加视图和菜单 # appbuilder.add_view(Repository_ModelView,"仓库",icon = 'fa-shopping-basket',category = '训练',category_icon = 'fa-sitemap') -# 添加api + class Repository_ModelView_Api(Repository_ModelView_Base,MyappModelRestApi): datamodel = SQLAInterface(Repository) @@ -173,8 +170,6 @@ class Images_Filter(MyappFilter): return result - -# 定义数据库视图 class Images_ModelView_Base(): label_title='镜像' datamodel = SQLAInterface(Images) @@ -193,22 +188,22 @@ class Images_ModelView_Base(): "dockerfile": StringField( _(datamodel.obj.lab('dockerfile')), description='镜像的构建Dockerfile全部内容', - widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象，以及widget函数的参数 + widget=MyBS3TextAreaFieldWidget(rows=10), ), "name": StringField( _(datamodel.obj.lab('name')), description='镜像名称全称，例如ubuntu:20.04', - widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象，以及widget函数的参数 + widget=BS3TextFieldWidget(), ), "entrypoint": StringField( _(datamodel.obj.lab('entrypoint')), description='镜像的入口命令，直接写成单行字符串，例如python xx.py，无需添加[]', - widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象，以及widget函数的参数 + widget=BS3TextFieldWidget(), ) } edit_form_extra_fields = add_form_extra_fields - base_filters = [["id", Images_Filter, lambda: []]] # 设置权限过滤器 + base_filters = [["id", Images_Filter, lambda: []]] @@ -218,7 +213,6 @@ class Images_ModelView(Images_ModelView_Base,MyappModelView,DeleteMixin): appbuilder.add_view(Images_ModelView,"模板镜像",href="/images_modelview/list/?_flt_2_name=",icon = 'fa-file-image-o',category = '训练') -# 添加api class Images_ModelView_Api(Images_ModelView_Base,MyappModelRestApi): datamodel = SQLAInterface(Images) route_base = '/images_modelview/api' @@ -226,7 +220,3 @@ class Images_ModelView_Api(Images_ModelView_Base,MyappModelRestApi): appbuilder.add_api(Images_ModelView_Api) - - - -appbuilder.add_separator("训练") # 在指定菜单栏下面的每个子菜单中间添加一个分割线的显示。 diff --git a/myapp/views/view_metadata.py b/myapp/views/view_metadata.py index a0a4cff9..e78aad1f 100644 --- a/myapp/views/view_metadata.py +++ b/myapp/views/view_metadata.py @@ -17,7 +17,6 @@ import re,os from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 from myapp import app, appbuilder,db,event_logger from myapp.utils import core from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList diff --git a/myapp/views/view_metadata_metric.py b/myapp/views/view_metadata_metric.py index 0abc7378..6f631392 100644 --- a/myapp/views/view_metadata_metric.py +++ b/myapp/views/view_metadata_metric.py @@ -16,7 +16,6 @@ import re,os from wtforms.validators import DataRequired, Length, NumberRange, Optional,Regexp from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 from myapp import app, appbuilder,db,event_logger from myapp.utils import core from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList @@ -109,7 +108,7 @@ class Metadata_metric_ModelView_base(): add_columns = ['app','metric_data_type','name','label','describe','metric_type','metric_level','metric_dim','metric_responsible','caliber','task_id','public'] # show_columns = ['project','name','describe','config_html','dag_json_html','created_by','changed_by','created_on','changed_on','expand_html'] edit_columns = add_columns - base_filters = [["id", Metadata_Metrics_table_Filter, lambda: []]] # 设置权限过滤器 + base_filters = [["id", Metadata_Metrics_table_Filter, lambda: []]] add_form_extra_fields = { "app": SelectField( label=_(datamodel.obj.lab('app')), diff --git a/myapp/views/view_pipeline.py b/myapp/views/view_pipeline.py index c48fa1a0..39f1e842 100644 --- a/myapp/views/view_pipeline.py +++ b/myapp/views/view_pipeline.py @@ -10,7 +10,7 @@ import re import urllib.parse from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 + from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob from myapp.models.model_team import Project,Project_User from myapp.views.view_team import Project_Join_Filter @@ -65,7 +65,7 @@ from flask import ( ) from myapp import security_manager from myapp.views.view_team import filter_join_org_project -import kfp # 使用自定义的就要把pip安装的删除了 +import kfp from werkzeug.datastructures import FileStorage from kubernetes import client as k8s_client from .base import ( @@ -113,7 +113,7 @@ class Pipeline_Filter(MyappFilter): from sqlalchemy.exc import InvalidRequestError,OperationalError -# 将定义pipeline的流程 +# 将dag 转为argo pipeline yaml # @pysnooper.snoop(watch_explode=()) def dag_to_pipeline(pipeline,dbsession,**kwargs): if not pipeline.id: diff --git a/myapp/views/view_task.py b/myapp/views/view_task.py index d6a3332c..3a5e041a 100644 --- a/myapp/views/view_task.py +++ b/myapp/views/view_task.py @@ -9,7 +9,6 @@ import uuid import re from kfp import compiler from sqlalchemy.exc import InvalidRequestError -# 将model添加成视图，并控制在前端的显示 from myapp.models.model_job import Repository,Images,Job_Template,Task,Pipeline,Workflow,Tfjob,Xgbjob,RunHistory,Pytorchjob from myapp.models.model_team import Project,Project_User from flask_appbuilder.actions import action @@ -57,7 +56,7 @@ from flask import ( url_for, ) from myapp import security_manager -import kfp # 使用自定义的就要把pip安装的删除了 + from werkzeug.datastructures import FileStorage from .base import ( api, @@ -117,7 +116,7 @@ class Task_ModelView_Base(): add_form_extra_fields = { "args": StringField( _(datamodel.obj.lab('args')), - widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象，以及widget函数的参数 + widget=MyBS3TextAreaFieldWidget(rows=10), ), "pipeline": QuerySelectField( datamodel.obj.lab('pipeline'), diff --git a/myapp/views/view_team.py b/myapp/views/view_team.py index 1bb845a3..a4a82c3d 100644 --- a/myapp/views/view_team.py +++ b/myapp/views/view_team.py @@ -4,7 +4,6 @@ from flask_appbuilder import ModelView, ModelRestApi from flask_appbuilder import ModelView,AppBuilder,expose,BaseView,has_access from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ -# 将model添加成视图，并控制在前端的显示 from myapp.models.model_team import Project,Project_User from flask_appbuilder.actions import action from wtforms import BooleanField, IntegerField, SelectField, StringField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList @@ -100,7 +99,7 @@ class Project_User_ModelView(Project_User_ModelView_Base,CompactCRUDMixin, Myapp appbuilder.add_view_no_menu(Project_User_ModelView) -# 添加api + class Project_User_ModelView_Api(Project_User_ModelView_Base,MyappModelRestApi): datamodel = SQLAInterface(Project_User) route_base = '/project_user_modelview/api' @@ -135,7 +134,7 @@ class Project_Join_Filter(MyappFilter): -# 获取查询自己所在的项目组的project +# query joined project def filter_join_org_project(): query = db.session.query(Project) # user_roles = [role.name.lower() for role in list(get_user_roles())] @@ -143,7 +142,6 @@ def filter_join_org_project(): if g.user.is_admin(): return query.filter(Project.type=='org').order_by(Project.id.desc()) - # 查询自己拥有的项目 my_user_id = g.user.get_id() if g.user else 0 owner_ids_query = db.session.query(Project_User.project_id).filter(Project_User.user_id == my_user_id) @@ -153,7 +151,7 @@ def filter_join_org_project(): class Project_ModelView_Base(): label_title='项目组' datamodel = SQLAInterface(Project) - base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些 + base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] base_order = ('id', 'desc') list_columns = ['name','user','type'] cols_width = { @@ -189,9 +187,7 @@ class Project_ModelView_Base(): if not g.user.username in item.get_creators(): raise MyappException('just creator can add/edit') - # 检测是否具有编辑权限，只有creator和admin可以编辑 - - # 打开编辑前，校验权限 + # before update, check permission def pre_update_get(self, item): self.pre_add_get() self.check_item_permissions(item) @@ -199,7 +195,7 @@ class Project_ModelView_Base(): flash('just creator can add/edit user','warning') raise MyappException('just creator can add/edit user') - # 对当前记录是否有权限 + # check permission def check_item_permissions(self,item): if g.user.is_admin() or g.user.username in item.get_creators(): self.user_permissions = { @@ -218,7 +214,7 @@ class Project_ModelView_Base(): - # 添加创始人 + # add project user def post_add(self, item): if not item.type: item.type = self.project_type @@ -235,19 +231,18 @@ class Project_ModelView_Base(): # class Project_ModelView_job_template(Project_ModelView_Base,MyappModelView): # project_type = 'job-template' -# base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器 +# base_filters = [["id", Project_Filter, project_type]] # datamodel = SQLAInterface(Project) # label_title = '模板分类' # # appbuilder.add_view(Project_ModelView_job_template,"模板分类",icon = 'fa-tasks',category = '项目组',category_icon = 'fa-users') -# 添加api class Project_ModelView_job_template_Api(Project_ModelView_Base,MyappModelRestApi): route_base = '/project_modelview/job_template/api' datamodel = SQLAInterface(Project) project_type = 'job-template' - base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器 + base_filters = [["id", Project_Filter, project_type]] related_views = [Project_User_ModelView_Api, ] label_title = '模板分类' edit_form_extra_fields={ @@ -271,19 +266,19 @@ appbuilder.add_api(Project_ModelView_job_template_Api) # class Project_ModelView_org(Project_ModelView_Base,MyappModelView): # project_type='org' -# base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器 +# base_filters = [["id", Project_Filter, project_type]] # datamodel = SQLAInterface(Project) # label_title = '项目分组' # # appbuilder.add_view(Project_ModelView_org,"项目分组",icon = 'fa-sitemap',category = '项目组',category_icon = 'fa-users') # -# 添加api + class Project_ModelView_org_Api(Project_ModelView_Base,MyappModelRestApi): route_base = '/project_modelview/org/api' datamodel = SQLAInterface(Project) project_type = 'org' - base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器 + base_filters = [["id", Project_Filter, project_type]] related_views = [Project_User_ModelView_Api, ] label_title = '项目分组' edit_form_extra_fields={ @@ -309,21 +304,20 @@ appbuilder.add_api(Project_ModelView_org_Api) # class Project_ModelView_train_model(Project_ModelView_Base,MyappModelView): # project_type = 'model' -# base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器 +# base_filters = [["id", Project_Filter, project_type]] # datamodel = SQLAInterface(Project) # label_title = '模型分组' # -# # 添加视图和菜单 # appbuilder.add_view(Project_ModelView_train_model,"模型分组",icon = 'fa-address-book-o',category = '项目组',category_icon = 'fa-users') # -# 添加api + class Project_ModelView_train_model_Api(Project_ModelView_Base,MyappModelRestApi): route_base = '/project_modelview/model/api' datamodel = SQLAInterface(Project) project_type = 'model' label_title = '模型分组' - base_filters = [["id", Project_Filter, project_type]] # 设置权限过滤器 + base_filters = [["id", Project_Filter, project_type]] related_views = [Project_User_ModelView_Api, ] edit_form_extra_fields={ 'type': StringField( @@ -340,7 +334,6 @@ appbuilder.add_api(Project_ModelView_train_model_Api) -# 添加api class Project_ModelView_Api(Project_ModelView_Base,MyappModelRestApi): datamodel = SQLAInterface(Project) route_base = '/project_modelview/api' diff --git a/myapp/views/view_train_model.py b/myapp/views/view_train_model.py index fa2d6c0a..dddc4423 100644 --- a/myapp/views/view_train_model.py +++ b/myapp/views/view_train_model.py @@ -1,7 +1,6 @@ from flask import render_template,redirect from flask_appbuilder.models.sqla.interface import SQLAInterface from flask import Blueprint, current_app, jsonify, make_response, request -# 将model添加成视图，并控制在前端的显示 from myapp.models.model_serving import Service from myapp.models.model_train_model import Training_Model from myapp.models.model_serving import InferenceService @@ -84,12 +83,10 @@ class Training_Model_Filter(MyappFilter): -# 定义数据库视图 -# class Training_Model_ModelView(JsonResModelView,DeleteMixin): class Training_Model_ModelView_Base(): datamodel = SQLAInterface(Training_Model) - base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] # 默认为这些 + base_permissions = ['can_add', 'can_edit', 'can_delete', 'can_list', 'can_show'] base_order = ('changed_on', 'desc') order_columns = ['id'] list_columns = ['project_url','name','version','framework','api_type','pipeline_url','creator','modified','deploy'] @@ -116,7 +113,7 @@ class Training_Model_ModelView_Base(): } label_title = '模型' - base_filters = [["id", Training_Model_Filter, lambda: []]] # 设置权限过滤器 + base_filters = [["id", Training_Model_Filter, lambda: []]] path_describe= r''' @@ -240,11 +237,10 @@ class Training_Model_ModelView_Base(): class Training_Model_ModelView(Training_Model_ModelView_Base,MyappModelView,DeleteMixin): datamodel = SQLAInterface(Training_Model) -# 添加视图和菜单 appbuilder.add_view(Training_Model_ModelView,"模型管理",icon = 'fa-hdd-o',category = '服务化',category_icon = 'fa-tasks') -# 添加api + class Training_Model_ModelView_Api(Training_Model_ModelView_Base,MyappModelRestApi): # noqa datamodel = SQLAInterface(Training_Model) # base_order = ('id', 'desc')