cube-studio/myapp/views/view_job_template.py

from myapp.views.baseSQLA import MyappSQLAInterface as SQLAInterface
from flask_babel import gettext as __
from flask_babel import lazy_gettext as _
import uuid
from wtforms.validators import DataRequired, Length, Regexp
import pysnooper
from sqlalchemy.exc import InvalidRequestError

from myapp.models.model_job import Job_Template
from flask_appbuilder.actions import action
from jinja2 import Environment, BaseLoader, DebugUndefined
from myapp.utils import core
from myapp import app, appbuilder, db

from wtforms import BooleanField, StringField, SelectField

from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget
from myapp.forms import MyBS3TextAreaFieldWidget, MyCodeArea
import logging
import re

from .baseApi import (
    MyappModelRestApi
)
from flask import (
    flash,
    g,
    Markup,
    make_response,
    redirect,
    request
)

from .base import (
    get_user_roles,
    MyappFilter,
)
from flask_appbuilder import expose
from myapp.views.view_team import Project_Filter,Creator_Filter
import datetime, time, json

conf = app.config


# 开发者能看到所有模板，用户只能看到release的模板
class Job_Tempalte_Filter(MyappFilter):
    # @pysnooper.snoop()
    def apply(self, query, func):
        user_roles = [role.name.lower() for role in list(self.get_user_roles())]
        if "admin" in user_roles:
            return query.order_by(self.model.id.desc())

        # join_projects_id = security_manager.get_join_projects_id(db.session)
        # logging.info(join_projects_id)
        return query.filter(self.model.version == 'Release').order_by(self.model.id.desc())


class Job_Template_ModelView_Base():
    datamodel = SQLAInterface(Job_Template)
    label_title = _('任务模板')
    check_redirect_list_url = conf.get('MODEL_URLS', {}).get('job_template', '')

    list_columns = ['project', 'name_title', 'version', 'creator', 'modified']
    spec_label_columns = {
        "project": _("功能分类")
    }
    cols_width = {
        "name_title": {"type": "ellip2", "width": 300},
        "name": {"type": "ellip2", "width": 400},
        "version": {"type": "ellip2", "width": 100},
        "modified": {"type": "ellip2", "width": 200},
    }
    show_columns = ['project', 'name', 'version', 'describe', 'images_url', 'workdir', 'entrypoint', 'args_html',
                    'demo_html', 'env', 'hostAliases', 'privileged', 'expand_html']
    add_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount',
                   'job_args_definition', 'args', 'env', 'hostAliases', 'privileged', 'accounts', 'demo', 'expand']
    edit_columns = add_columns

    base_filters = [["id", Job_Tempalte_Filter, lambda: []]]
    base_order = ('id', 'desc')
    order_columns = ['id']
    add_form_query_rel_fields = {
        "images": [["name", Creator_Filter, None]],
        "project": [["name", Project_Filter, 'job-template']],
    }
    version_list = [[version, version] for version in ['Alpha', 'Release']]
    edit_form_query_rel_fields = add_form_query_rel_fields
    add_form_extra_fields = {
        "name": StringField(
            _('名称'),
            description= _('英文名(小写字母、数字、- 组成)，最长50个字符'),
            default='',
            widget=BS3TextFieldWidget(),  # 传给widget函数的是外层的field对象，以及widget函数的参数
            validators=[Regexp("^[a-z][a-z0-9\-]*[a-z0-9]$"), Length(1, 54)]
        ),
        "describe": StringField(
            _("描述"),
            description= _("模板的描述将直接显示在pipeline编排界面"),
            default='',
            widget=BS3TextFieldWidget(),
            validators=[DataRequired()]
        ),
        "version": SelectField(
            _('版本'),
            description= _("job模板的版本，release版本的模板才能被所有用户看到"),
            default='',
            widget=Select2Widget(),
            choices=version_list
        ),
        "volume_mount": StringField(
            _('挂载'),
            default='',
            description= _('使用该模板的task，会在添加时，自动添加该挂载。<br>外部挂载，格式示例:$pvc_name1(pvc):/$container_path1,$hostpath1(hostpath):/$container_path2,4G(memory):/dev/shm,注意pvc会自动挂载对应目录下的个人rtx子目录'),
            widget=BS3TextFieldWidget(),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "workdir": StringField(
            _('工作目录'),
            description= _('工作目录，不填写将直接使用镜像默认的工作目录'),
            widget=BS3TextFieldWidget(),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "entrypoint": StringField(
            _('启动命令'),
            description= _('镜像的入口命令，直接写成单行字符串，例如python xx.py，无需添加[]'),
            default='',
            widget=BS3TextFieldWidget(),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "job_args_definition": StringField(
            _('参数定义'),
            description= _('使用job模板参数的标准填写方式'),
            widget=MyCodeArea(code=core.job_template_args_definition()),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "env": StringField(
            _('环境变量'),
            default='',
            description= _('使用模板的task自动添加的环境变量，支持模板变量。<br>书写格式:每行一个环境变量env_key=env_value'),
            widget=MyBS3TextAreaFieldWidget(rows=3),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "hostAliases": StringField(
            _('域名解析'),
            default='',
            description= _('添加到容器内的host映射。<br>书写格式:每行一个dns解析记录，ip host1 host2，<br>示例：1.1.1.1 example1.oa.com example2.oa.com'),
            widget=MyBS3TextAreaFieldWidget(rows=3),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "demo": StringField(
            _('demo'),
            description= _('填写demo'),
            widget=MyBS3TextAreaFieldWidget(rows=10),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "accounts": StringField(
            _('k8s账号'),
            default='',
            description= _('k8s的ServiceAccount，在此类任务运行时会自动挂载此账号，多用于模板用于k8s pod/cr时使用'),
            widget=BS3TextFieldWidget(),  # 传给widget函数的是外层的field对象，以及widget函数的参数
            validators=[]
        ),
        "privileged": BooleanField(
            _('超级权限'),
            description= _('是否启动超级权限')
        ),
        "expand": StringField(
            _('扩展'),
            default=json.dumps({"index": 0, "help_url": conf.get('DOCUMENTATION_URL')}, ensure_ascii=False, indent=4),
            description= _('json格式的扩展字段，支持<br> "index":"$模板展示顺序号"，<br>"help_url":"$帮助文档地址"，<br>"HostNetwork":true 启动主机端口监听'),
            widget=MyBS3TextAreaFieldWidget(rows=3),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        )
    }
    edit_form_extra_fields = add_form_extra_fields

    def set_columns(self,job_template=None):
        args_field = StringField(
            _('参数'),
            default=json.dumps({
                __("参数分组1"): {
                    "--attr1": {
                        "type": "str",
                        "label": __("参数1"),
                        "default": "value1",
                        "describe": __("这里是这个参数的描述和备注"),
                    }
                }
            }, indent=4, ensure_ascii=False),
            description= _('json格式，此类task使用时需要填写的参数，示例：')+'<br><pre><code>%s</code></pre>' % core.job_template_args_definition(),
            widget=MyBS3TextAreaFieldWidget(rows=10),  # 传给widget函数的是外层的field对象，以及widget函数的参数
            validators=[DataRequired()]
        )

        self.edit_form_extra_fields['args'] = args_field
        self.add_form_extra_fields['args'] = args_field

    pre_add_web = set_columns
    pre_update_web = set_columns

    # 校验是否是json
    # @pysnooper.snoop(watch_explode=('job_args'))
    def pre_add(self, item):
        if not item.env:
            item.env = ''
        envs = item.env.strip().split('\n')
        envs = [env.strip() for env in envs if env.strip() and '=' in env]
        item.env = '\n'.join(envs)
        if not item.args:
            item.args = '{}'
        item.args = core.validate_job_args(item)

        if not item.expand or not item.expand.strip():
            item.expand = '{}'
        core.validate_json(item.expand)
        item.expand = json.dumps(json.loads(item.expand), indent=4, ensure_ascii=False)

        if not item.demo or not item.demo.strip():
            item.demo = '{}'

        core.validate_json(item.demo)

        if item.hostAliases:
            # if not item.images.entrypoint:
            #     raise MyappException('images entrypoint not exist')
            all_host = {}
            all_rows = re.split('\r|\n', item.hostAliases)
            all_rows = [all_row.strip() for all_row in all_rows if all_row.strip()]
            for row in all_rows:
                hosts = row.split(' ')
                hosts = [host for host in hosts if host]
                if len(hosts) > 1:
                    if hosts[0] in all_host:
                        all_host[hosts[0]] = all_host[hosts[0]] + hosts[1:]
                    else:
                        all_host[hosts[0]] = hosts[1:]

            hostAliases = ''
            for ip in all_host:
                hostAliases += ip + " " + " ".join(all_host[ip])
                hostAliases += '\n'
            item.hostAliases = hostAliases.strip()

        task_args = json.loads(item.demo)
        job_args = json.loads(item.args)
        item.demo = json.dumps(core.validate_task_args(task_args, job_args), indent=4, ensure_ascii=False)

    # 检测是否具有编辑权限，只有creator和admin可以编辑
    def check_edit_permission(self, item):
        user_roles = [role.name.lower() for role in list(get_user_roles())]
        if "admin" in user_roles:
            return True
        if g.user and g.user.username and hasattr(item, 'created_by'):
            if g.user.username == item.created_by.username:
                return True
        flash('just creator can edit/delete ', 'warning')
        return False

    def pre_update(self, item):
        self.pre_add(item)

    # @pysnooper.snoop()
    def post_list(self,items):
        def sort_expand_index(items):
            all = {
                0: []
            }
            for item in items:
                try:
                    if item.expand:
                        index = float(json.loads(item.expand).get('index', 0))+float(json.loads(item.project.expand).get('index', 0))*1000
                        if index:
                            if index in all:
                                all[index].append(item)
                            else:
                                all[index] = [item]
                        else:
                            all[0].append(item)
                    else:
                        all[0].append(item)
                except Exception as e:
                    print(e)
            back = []
            for index in sorted(all):
                back.extend(all[index])
                # 当有小数的时候自动转正
                # if float(index)!=int(index):
                #     pass
            return back
        return sort_expand_index(items)



    @action("copy", "复制", confirmation= '复制所选记录?', icon="fa-copy",multiple=True, single=False)
    def copy(self, job_templates):
        if not isinstance(job_templates, list):
            job_templates = [job_templates]
        try:
            for job_template in job_templates:
                new_job_template = job_template.clone()
                new_job_template.name = new_job_template.name + "_copy_" + uuid.uuid4().hex[:4]
                new_job_template.created_on = datetime.datetime.now()
                new_job_template.changed_on = datetime.datetime.now()
                db.session.add(new_job_template)
                db.session.commit()
        except InvalidRequestError:
            db.session.rollback()
        except Exception as e:
            raise e
        return redirect(request.referrer)

# 添加api
class Job_Template_ModelView_Api(Job_Template_ModelView_Base, MyappModelRestApi):
    datamodel = SQLAInterface(Job_Template)
    page_size = 1000
    route_base = '/job_template_modelview/api'
    # add_columns = ['project', 'images', 'name', 'version', 'describe', 'args', 'env','hostAliases', 'privileged','accounts', 'demo','expand']
    add_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'args',
                   'env', 'hostAliases', 'privileged', 'accounts', 'expand']
    edit_columns = add_columns
    # list_columns = ['project','name','version','creator','modified']
    list_columns = ['project', 'name', 'version', 'describe', 'images', 'workdir', 'entrypoint', 'args', 'demo', 'env',
                    'hostAliases', 'privileged', 'accounts', 'created_by', 'changed_by', 'created_on', 'changed_on',
                    'expand']
    show_columns = ['project', 'name', 'version', 'describe', 'images', 'workdir', 'entrypoint', 'args', 'demo', 'env',
                    'hostAliases', 'privileged', 'expand']


appbuilder.add_api(Job_Template_ModelView_Api)


class Job_Template_fab_ModelView_Api(Job_Template_ModelView_Base, MyappModelRestApi):
    datamodel = SQLAInterface(Job_Template)
    route_base = '/job_template_fab_modelview/api'
    # add_columns = ['project', 'images', 'name', 'version', 'describe', 'args', 'env','hostAliases', 'privileged','accounts', 'demo','expand']
    add_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'args',
                   'env', 'hostAliases', 'privileged', 'accounts', 'expand']
    page_size = 1000
    edit_columns = add_columns
    list_columns = ['project', 'name', 'version', 'creator', 'modified']
    show_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'args',
                    'env', 'hostAliases', 'privileged', 'accounts', 'expand']

appbuilder.add_api(Job_Template_fab_ModelView_Api)