from myapp.views.baseSQLA import MyappSQLAInterface as SQLAInterface from flask_babel import gettext as __ from flask_babel import lazy_gettext as _ import uuid from wtforms.validators import DataRequired, Length, Regexp import pysnooper from sqlalchemy.exc import InvalidRequestError from myapp.models.model_job import Job_Template from flask_appbuilder.actions import action from jinja2 import Environment, BaseLoader, DebugUndefined from myapp.utils import core from myapp import app, appbuilder, db from wtforms import BooleanField, StringField, SelectField from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2Widget from myapp.forms import MyBS3TextAreaFieldWidget, MyCodeArea import logging import re from .baseApi import ( MyappModelRestApi ) from flask import ( flash, g, Markup, make_response, redirect, request ) from .base import ( get_user_roles, MyappFilter, ) from flask_appbuilder import expose from myapp.views.view_team import Project_Filter,Creator_Filter import datetime, time, json conf = app.config # 开发者能看到所有模板,用户只能看到release的模板 class Job_Tempalte_Filter(MyappFilter): # @pysnooper.snoop() def apply(self, query, func): user_roles = [role.name.lower() for role in list(self.get_user_roles())] if "admin" in user_roles: return query.order_by(self.model.id.desc()) # join_projects_id = security_manager.get_join_projects_id(db.session) # logging.info(join_projects_id) return query.filter(self.model.version == 'Release').order_by(self.model.id.desc()) class Job_Template_ModelView_Base(): datamodel = SQLAInterface(Job_Template) label_title = _('任务模板') check_redirect_list_url = conf.get('MODEL_URLS', {}).get('job_template', '') list_columns = ['project', 'name_title', 'version', 'creator', 'modified'] spec_label_columns = { "project": _("功能分类") } cols_width = { "name_title": {"type": "ellip2", "width": 300}, "name": {"type": "ellip2", "width": 400}, "version": {"type": "ellip2", "width": 100}, "modified": {"type": "ellip2", "width": 200}, } show_columns = ['project', 'name', 'version', 'describe', 'images_url', 'workdir', 'entrypoint', 'args_html', 'demo_html', 'env', 'hostAliases', 'privileged', 'expand_html'] add_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'job_args_definition', 'args', 'env', 'hostAliases', 'privileged', 'accounts', 'demo', 'expand'] edit_columns = add_columns base_filters = [["id", Job_Tempalte_Filter, lambda: []]] base_order = ('id', 'desc') order_columns = ['id'] add_form_query_rel_fields = { "images": [["name", Creator_Filter, None]], "project": [["name", Project_Filter, 'job-template']], } version_list = [[version, version] for version in ['Alpha', 'Release']] edit_form_query_rel_fields = add_form_query_rel_fields add_form_extra_fields = { "name": StringField( _('名称'), description= _('英文名(小写字母、数字、- 组成),最长50个字符'), default='', widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象,以及widget函数的参数 validators=[Regexp("^[a-z][a-z0-9\-]*[a-z0-9]$"), Length(1, 54)] ), "describe": StringField( _("描述"), description= _("模板的描述将直接显示在pipeline编排界面"), default='', widget=BS3TextFieldWidget(), validators=[DataRequired()] ), "version": SelectField( _('版本'), description= _("job模板的版本,release版本的模板才能被所有用户看到"), default='', widget=Select2Widget(), choices=version_list ), "volume_mount": StringField( _('挂载'), default='', description= _('使用该模板的task,会在添加时,自动添加该挂载。
外部挂载,格式示例:$pvc_name1(pvc):/$container_path1,$hostpath1(hostpath):/$container_path2,4G(memory):/dev/shm,注意pvc会自动挂载对应目录下的个人rtx子目录'), widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "workdir": StringField( _('工作目录'), description= _('工作目录,不填写将直接使用镜像默认的工作目录'), widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "entrypoint": StringField( _('启动命令'), description= _('镜像的入口命令,直接写成单行字符串,例如python xx.py,无需添加[]'), default='', widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "job_args_definition": StringField( _('参数定义'), description= _('使用job模板参数的标准填写方式'), widget=MyCodeArea(code=core.job_template_args_definition()), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "env": StringField( _('环境变量'), default='', description= _('使用模板的task自动添加的环境变量,支持模板变量。
书写格式:每行一个环境变量env_key=env_value'), widget=MyBS3TextAreaFieldWidget(rows=3), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "hostAliases": StringField( _('域名解析'), default='', description= _('添加到容器内的host映射。
书写格式:每行一个dns解析记录,ip host1 host2,
示例:1.1.1.1 example1.oa.com example2.oa.com'), widget=MyBS3TextAreaFieldWidget(rows=3), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "demo": StringField( _('demo'), description= _('填写demo'), widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象,以及widget函数的参数 ), "accounts": StringField( _('k8s账号'), default='', description= _('k8s的ServiceAccount,在此类任务运行时会自动挂载此账号,多用于模板用于k8s pod/cr时使用'), widget=BS3TextFieldWidget(), # 传给widget函数的是外层的field对象,以及widget函数的参数 validators=[] ), "privileged": BooleanField( _('超级权限'), description= _('是否启动超级权限') ), "expand": StringField( _('扩展'), default=json.dumps({"index": 0, "help_url": conf.get('DOCUMENTATION_URL')}, ensure_ascii=False, indent=4), description= _('json格式的扩展字段,支持
"index":"$模板展示顺序号",
"help_url":"$帮助文档地址",
"HostNetwork":true 启动主机端口监听'), widget=MyBS3TextAreaFieldWidget(rows=3), # 传给widget函数的是外层的field对象,以及widget函数的参数 ) } edit_form_extra_fields = add_form_extra_fields def set_columns(self,job_template=None): args_field = StringField( _('参数'), default=json.dumps({ __("参数分组1"): { "--attr1": { "type": "str", "label": __("参数1"), "default": "value1", "describe": __("这里是这个参数的描述和备注"), } } }, indent=4, ensure_ascii=False), description= _('json格式,此类task使用时需要填写的参数,示例:')+'
%s
' % core.job_template_args_definition(), widget=MyBS3TextAreaFieldWidget(rows=10), # 传给widget函数的是外层的field对象,以及widget函数的参数 validators=[DataRequired()] ) self.edit_form_extra_fields['args'] = args_field self.add_form_extra_fields['args'] = args_field pre_add_web = set_columns pre_update_web = set_columns # 校验是否是json # @pysnooper.snoop(watch_explode=('job_args')) def pre_add(self, item): if not item.env: item.env = '' envs = item.env.strip().split('\n') envs = [env.strip() for env in envs if env.strip() and '=' in env] item.env = '\n'.join(envs) if not item.args: item.args = '{}' item.args = core.validate_job_args(item) if not item.expand or not item.expand.strip(): item.expand = '{}' core.validate_json(item.expand) item.expand = json.dumps(json.loads(item.expand), indent=4, ensure_ascii=False) if not item.demo or not item.demo.strip(): item.demo = '{}' core.validate_json(item.demo) if item.hostAliases: # if not item.images.entrypoint: # raise MyappException('images entrypoint not exist') all_host = {} all_rows = re.split('\r|\n', item.hostAliases) all_rows = [all_row.strip() for all_row in all_rows if all_row.strip()] for row in all_rows: hosts = row.split(' ') hosts = [host for host in hosts if host] if len(hosts) > 1: if hosts[0] in all_host: all_host[hosts[0]] = all_host[hosts[0]] + hosts[1:] else: all_host[hosts[0]] = hosts[1:] hostAliases = '' for ip in all_host: hostAliases += ip + " " + " ".join(all_host[ip]) hostAliases += '\n' item.hostAliases = hostAliases.strip() task_args = json.loads(item.demo) job_args = json.loads(item.args) item.demo = json.dumps(core.validate_task_args(task_args, job_args), indent=4, ensure_ascii=False) # 检测是否具有编辑权限,只有creator和admin可以编辑 def check_edit_permission(self, item): user_roles = [role.name.lower() for role in list(get_user_roles())] if "admin" in user_roles: return True if g.user and g.user.username and hasattr(item, 'created_by'): if g.user.username == item.created_by.username: return True flash('just creator can edit/delete ', 'warning') return False def pre_update(self, item): self.pre_add(item) # @pysnooper.snoop() def post_list(self,items): def sort_expand_index(items): all = { 0: [] } for item in items: try: if item.expand: index = float(json.loads(item.expand).get('index', 0))+float(json.loads(item.project.expand).get('index', 0))*1000 if index: if index in all: all[index].append(item) else: all[index] = [item] else: all[0].append(item) else: all[0].append(item) except Exception as e: print(e) back = [] for index in sorted(all): back.extend(all[index]) # 当有小数的时候自动转正 # if float(index)!=int(index): # pass return back return sort_expand_index(items) @action("copy", "复制", confirmation= '复制所选记录?', icon="fa-copy",multiple=True, single=False) def copy(self, job_templates): if not isinstance(job_templates, list): job_templates = [job_templates] try: for job_template in job_templates: new_job_template = job_template.clone() new_job_template.name = new_job_template.name + "_copy_" + uuid.uuid4().hex[:4] new_job_template.created_on = datetime.datetime.now() new_job_template.changed_on = datetime.datetime.now() db.session.add(new_job_template) db.session.commit() except InvalidRequestError: db.session.rollback() except Exception as e: raise e return redirect(request.referrer) # 添加api class Job_Template_ModelView_Api(Job_Template_ModelView_Base, MyappModelRestApi): datamodel = SQLAInterface(Job_Template) page_size = 1000 route_base = '/job_template_modelview/api' # add_columns = ['project', 'images', 'name', 'version', 'describe', 'args', 'env','hostAliases', 'privileged','accounts', 'demo','expand'] add_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'args', 'env', 'hostAliases', 'privileged', 'accounts', 'expand'] edit_columns = add_columns # list_columns = ['project','name','version','creator','modified'] list_columns = ['project', 'name', 'version', 'describe', 'images', 'workdir', 'entrypoint', 'args', 'demo', 'env', 'hostAliases', 'privileged', 'accounts', 'created_by', 'changed_by', 'created_on', 'changed_on', 'expand'] show_columns = ['project', 'name', 'version', 'describe', 'images', 'workdir', 'entrypoint', 'args', 'demo', 'env', 'hostAliases', 'privileged', 'expand'] appbuilder.add_api(Job_Template_ModelView_Api) class Job_Template_fab_ModelView_Api(Job_Template_ModelView_Base, MyappModelRestApi): datamodel = SQLAInterface(Job_Template) route_base = '/job_template_fab_modelview/api' # add_columns = ['project', 'images', 'name', 'version', 'describe', 'args', 'env','hostAliases', 'privileged','accounts', 'demo','expand'] add_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'args', 'env', 'hostAliases', 'privileged', 'accounts', 'expand'] page_size = 1000 edit_columns = add_columns list_columns = ['project', 'name', 'version', 'creator', 'modified'] show_columns = ['project', 'images', 'name', 'version', 'describe', 'workdir', 'entrypoint', 'volume_mount', 'args', 'env', 'hostAliases', 'privileged', 'accounts', 'expand'] appbuilder.add_api(Job_Template_fab_ModelView_Api)