cube-studio/myapp/views/view_service_pipeline.py

from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_babel import lazy_gettext as _

from myapp.models.model_service_pipeline import Service_Pipeline
from myapp.models.model_job import Repository
from flask import jsonify
from flask_appbuilder.forms import GeneralModelConverter
from myapp.utils import core
from myapp import app, appbuilder, db
from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import DataRequired, Length, Regexp
from sqlalchemy import or_
from myapp.exceptions import MyappException
from wtforms import StringField, SelectField
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget, Select2ManyWidget, Select2Widget
from myapp.forms import MyBS3TextAreaFieldWidget, MySelectMultipleField
import copy
from .baseApi import (
    MyappModelRestApi
)
from flask import (
    flash,
    g,
    make_response,
    redirect,
    request,
)
from myapp import security_manager
from myapp.views.view_team import filter_join_org_project
from .base import (
    DeleteMixin,
    get_user_roles,
    MyappFilter,
    MyappModelView,
    json_response
)

from flask_appbuilder import expose
import pysnooper, datetime, json

conf = app.config
logging = app.logger


class Service_Pipeline_Filter(MyappFilter):
    # @pysnooper.snoop()
    def apply(self, query, func):
        user_roles = [role.name.lower() for role in list(self.get_user_roles())]
        if "admin" in user_roles:
            return query

        join_projects_id = security_manager.get_join_projects_id(db.session)
        # public_project_id =
        # logging.info(join_projects_id)
        return query.filter(
            or_(
                self.model.project_id.in_(join_projects_id),
                # self.model.project.name.in_(['public'])
            )
        )


class Service_Pipeline_ModelView_Base():
    label_title = '任务流'
    datamodel = SQLAInterface(Service_Pipeline)
    check_redirect_list_url = conf.get('MODEL_URLS', {}).get('service_pipeline', '')

    base_permissions = ['can_show', 'can_edit', 'can_list', 'can_delete', 'can_add']
    base_order = ("changed_on", "desc")
    # order_columns = ['id','changed_on']
    order_columns = ['id']

    list_columns = ['project', 'service_pipeline_url', 'creator', 'modified', 'operate_html']
    add_columns = ['project', 'name', 'describe', 'namespace', 'images', 'env', 'resource_memory', 'resource_cpu',
                   'resource_gpu', 'replicas', 'dag_json', 'alert_status', 'alert_user', 'parameter']
    show_columns = ['project', 'name', 'describe', 'namespace', 'run_id', 'created_by', 'changed_by', 'created_on',
                    'changed_on', 'expand_html', 'parameter_html']
    edit_columns = add_columns

    base_filters = [["id", Service_Pipeline_Filter, lambda: []]]
    conv = GeneralModelConverter(datamodel)

    add_form_extra_fields = {

        "name": StringField(
            _(datamodel.obj.lab('name')),
            description="英文名(小写字母、数字、- 组成)，最长50个字符",
            widget=BS3TextFieldWidget(),
            validators=[Regexp("^[a-z][a-z0-9\-]*[a-z0-9]$"), Length(1, 54), DataRequired()]
        ),
        "project": QuerySelectField(
            _(datamodel.obj.lab('project')),
            query_factory=filter_join_org_project,
            allow_blank=True,
            widget=Select2Widget()
        ),
        "dag_json": StringField(
            _(datamodel.obj.lab('dag_json')),
            widget=MyBS3TextAreaFieldWidget(rows=10),  # 传给widget函数的是外层的field对象，以及widget函数的参数
        ),
        "namespace": StringField(
            _(datamodel.obj.lab('namespace')),
            description="部署task所在的命名空间(目前无需填写)",
            default='service',
            widget=BS3TextFieldWidget()
        ),
        "images": StringField(
            _(datamodel.obj.lab('images')),
            default='ccr.ccs.tencentyun.com/cube-studio/service-pipeline',
            description="推理服务镜像",
            widget=BS3TextFieldWidget(),
            validators=[DataRequired()]
        ),
        "node_selector": StringField(
            _(datamodel.obj.lab('node_selector')),
            description="部署task所在的机器(目前无需填写)",
            widget=BS3TextFieldWidget(),
            default=datamodel.obj.node_selector.default.arg
        ),
        "image_pull_policy": SelectField(
            _(datamodel.obj.lab('image_pull_policy')),
            description="镜像拉取策略(always为总是拉取远程镜像，IfNotPresent为若本地存在则使用本地镜像)",
            widget=Select2Widget(),
            choices=[['Always', 'Always'], ['IfNotPresent', 'IfNotPresent']]
        ),

        "alert_status": MySelectMultipleField(
            label=_(datamodel.obj.lab('alert_status')),
            widget=Select2ManyWidget(),
            choices=[[x, x] for x in
                     ['Created', 'Pending', 'Running', 'Succeeded', 'Failed', 'Unknown', 'Waiting', 'Terminated']],
            description="选择通知状态"
        ),
        "alert_user": StringField(
            label=_(datamodel.obj.lab('alert_user')),
            widget=BS3TextFieldWidget(),
            description="选择通知用户，每个用户使用逗号分隔"
        ),

        "label": StringField(_(datamodel.obj.lab('label')), description='中文名', widget=BS3TextFieldWidget(),
                             validators=[DataRequired()]),
        "resource_memory": StringField(_(datamodel.obj.lab('resource_memory')),
                                       default=Service_Pipeline.resource_memory.default.arg,
                                       description='内存的资源使用限制，示例1G，10G， 最大100G，如需更多联系管理员', widget=BS3TextFieldWidget(),
                                       validators=[DataRequired()]),
        "resource_cpu": StringField(_(datamodel.obj.lab('resource_cpu')), default=Service_Pipeline.resource_cpu.default.arg,
                                    description='cpu的资源使用限制(单位核)，示例 0.4，10，最大50核，如需更多联系管理员',
                                    widget=BS3TextFieldWidget(), validators=[DataRequired()]),
        "resource_gpu": StringField(_(datamodel.obj.lab('resource_gpu')), default='0',
                                    description='gpu的资源使用限制(单位卡)，示例:1，2，训练任务每个容器独占整卡。申请具体的卡型号，可以类似 1(V100),目前支持T4/V100/A100/VGPU',
                                    widget=BS3TextFieldWidget()),
        "replicas": StringField(_(datamodel.obj.lab('replicas')), default=Service_Pipeline.replicas.default.arg,
                                description='pod副本数，用来配置高可用', widget=BS3TextFieldWidget(), validators=[DataRequired()]),
        "env": StringField(_(datamodel.obj.lab('env')), default=Service_Pipeline.env.default.arg,
                           description='使用模板的task自动添加的环境变量，支持模板变量。书写格式:每行一个环境变量env_key=env_value',
                           widget=MyBS3TextAreaFieldWidget()),

    }

    edit_form_extra_fields = add_form_extra_fields

    # 检测是否具有编辑权限，只有creator和admin可以编辑
    def check_edit_permission(self, item):
        user_roles = [role.name.lower() for role in list(get_user_roles())]
        if "admin" in user_roles:
            return True
        if g.user and g.user.username and hasattr(item, 'created_by'):
            if g.user.username == item.created_by.username:
                return True
        flash('just creator can edit/delete ', 'warning')
        return False

    # 验证args参数
    @pysnooper.snoop(watch_explode=('item'))
    def service_pipeline_args_check(self, item):
        core.validate_str(item.name, 'name')
        if not item.dag_json:
            item.dag_json = '{}'
        core.validate_json(item.dag_json)

        # 校验task的关系，没有闭环，并且顺序要对。没有配置的，自动没有上游，独立
        # @pysnooper.snoop()
        def order_by_upstream(dag_json):
            order_dag = {}
            tasks_name = list(dag_json.keys())  # 如果没有配全的话，可能只有局部的task
            i = 0
            while tasks_name:
                i += 1
                if i > 100:  # 不会有100个依赖关系
                    break
                for task_name in tasks_name:
                    # 没有上游的情况
                    if not dag_json[task_name]:
                        order_dag[task_name] = dag_json[task_name]
                        tasks_name.remove(task_name)
                        continue
                    # 没有上游的情况
                    elif 'upstream' not in dag_json[task_name] or not dag_json[task_name]['upstream']:
                        order_dag[task_name] = dag_json[task_name]
                        tasks_name.remove(task_name)
                        continue
                    # 如果有上游依赖的话，先看上游任务是否已经加到里面了。
                    upstream_all_ready = True
                    for upstream_task_name in dag_json[task_name]['upstream']:
                        if upstream_task_name not in order_dag:
                            upstream_all_ready = False
                    if upstream_all_ready:
                        order_dag[task_name] = dag_json[task_name]
                        tasks_name.remove(task_name)
                    else:
                        dag_json[task_name]['upstream'] = []
                        order_dag[task_name] = dag_json[task_name]
                        tasks_name.remove(task_name)

            if list(dag_json.keys()).sort() != list(order_dag.keys()).sort():
                flash('dag service pipeline 存在循环或未知上游', category='warning')
                raise MyappException('dag service pipeline 存在循环或未知上游')
            return order_dag

        # 配置上缺少的默认上游
        dag_json = json.loads(item.dag_json)
        item.dag_json = json.dumps(order_by_upstream(copy.deepcopy(dag_json)), ensure_ascii=False, indent=4)

        # raise Exception('args is not valid')

    # @pysnooper.snoop()
    def pre_add(self, item):
        item.name = item.name.replace('_', '-')[0:54].lower().strip('-')
        # item.alert_status = ','.join(item.alert_status)
        # self.service_pipeline_args_check(item)
        item.create_datetime = datetime.datetime.now()
        item.change_datetime = datetime.datetime.now()
        item.parameter = json.dumps({}, indent=4, ensure_ascii=False)
        item.volume_mount = item.project.volume_mount + ",%s(configmap):/config/" % item.name

    # @pysnooper.snoop()
    def pre_update(self, item):
        if item.expand:
            core.validate_json(item.expand)
            item.expand = json.dumps(json.loads(item.expand), indent=4, ensure_ascii=False)
        else:
            item.expand = '{}'
        item.name = item.name.replace('_', '-')[0:54].lower()
        item.alert_status = ','.join(item.alert_status)
        # self.service_pipeline_args_check(item)
        item.change_datetime = datetime.datetime.now()
        item.parameter = json.dumps(json.loads(item.parameter),indent=4,ensure_ascii=False) if item.parameter else '{}'
        item.dag_json = json.dumps(json.loads(item.dag_json), indent=4,ensure_ascii=False) if item.dag_json else '{}'
        item.volume_mount = item.project.volume_mount + ",%s(configmap):/config/" % item.name

    @expose("/my/list/")
    def my(self):
        try:
            user_id = g.user.id
            if user_id:
                service_pipelines = db.session.query(Service_Pipeline).filter_by(created_by_fk=user_id).all()
                back = []
                for service_pipeline in service_pipelines:
                    back.append(service_pipeline.to_json())
                return json_response(message='success', status=0, result=back)
        except Exception as e:
            print(e)
            return json_response(message=str(e), status=-1, result={})

    def check_service_pipeline_perms(user_fun):
        # @pysnooper.snoop()
        def wraps(*args, **kwargs):
            service_pipeline_id = int(kwargs.get('service_pipeline_id', '0'))
            if not service_pipeline_id:
                response = make_response("service_pipeline_id not exist")
                response.status_code = 404
                return response

            user_roles = [role.name.lower() for role in g.user.roles]
            if "admin" in user_roles:
                return user_fun(*args, **kwargs)

            join_projects_id = security_manager.get_join_projects_id(db.session)
            service_pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first()
            if service_pipeline.project.id in join_projects_id:
                return user_fun(*args, **kwargs)

            response = make_response("no perms to run pipeline %s" % service_pipeline_id)
            response.status_code = 403
            return response

        return wraps

    # 构建同步服务
    def build_http(self, service_pipeline):
        pass

    # 构建异步服务
    @pysnooper.snoop()
    def build_mq_consumer(self, service_pipeline):
        namespace = conf.get('SERVICE_PIPELINE_NAMESPACE')
        name = service_pipeline.name
        command = service_pipeline.command

        image_pull_secrets = conf.get('HUBSECRET', [])
        user_repositorys = db.session.query(Repository).filter(Repository.created_by_fk == g.user.id).all()
        image_pull_secrets = list(set(image_pull_secrets + [rep.hubsecret for rep in user_repositorys]))

        from myapp.utils.py.py_k8s import K8s
        k8s_client = K8s(service_pipeline.project.cluster.get('KUBECONFIG', ''))
        dag_json = service_pipeline.dag_json if service_pipeline.dag_json else '{}'

        # 生成服务使用的configmap

        config_data = {
            "dag.json": dag_json
        }
        k8s_client.create_configmap(namespace=namespace, name=name, data=config_data, labels={'app': name})
        env = service_pipeline.env
        if conf.get('SERVICE_PIPELINE_JAEGER', ''):
            env['JAEGER_HOST'] = conf.get('SERVICE_PIPELINE_JAEGER', '')
            env['SERVICE_NAME'] = name

        labels = {"app": name, "user": service_pipeline.created_by.username, "pod-type": "service-pipeline"}
        k8s_client.create_deployment(
            namespace=namespace,
            name=name,
            replicas=service_pipeline.replicas,
            labels=labels,
            # command=['sh','-c',command] if command else None,
            command=['bash', '-c', "python mq-pipeline/cube_kafka.py"],
            args=None,
            volume_mount=service_pipeline.volume_mount,
            working_dir=service_pipeline.working_dir,
            node_selector=service_pipeline.get_node_selector(),
            resource_memory=service_pipeline.resource_memory,
            resource_cpu=service_pipeline.resource_cpu,
            resource_gpu=service_pipeline.resource_gpu if service_pipeline.resource_gpu else '',
            image_pull_policy=conf.get('IMAGE_PULL_POLICY', 'Always'),
            image_pull_secrets=image_pull_secrets,
            image=service_pipeline.images,
            hostAliases=conf.get('HOSTALIASES', ''),
            env=env,
            privileged=False,
            accounts=None,
            username=service_pipeline.created_by.username,
            ports=None
        )

        pass

    # 只能有一个入口。不能同时接口两个队列
    # # @event_logger.log_this
    @expose("/run_service_pipeline/<service_pipeline_id>", methods=["GET", "POST"])
    @check_service_pipeline_perms
    def run_service_pipeline(self, service_pipeline_id):
        service_pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first()
        dag_json = json.loads(service_pipeline.dag_json)
        root_nodes_name = service_pipeline.get_root_node_name()
        self.clear(service_pipeline_id)
        if root_nodes_name:
            root_node_name = root_nodes_name[0]
            root_node = dag_json[root_node_name]
            # 构建异步
            if root_node['template-group'] == 'endpoint' and root_node['template'] == 'mq':
                self.build_mq_consumer(service_pipeline)

            # 构建同步
            if root_node['template-group'] == 'endpoint' and root_node['template'] == 'gateway':
                self.build_http(service_pipeline)

        return redirect("/service_pipeline_modelview/web/log/%s" % service_pipeline_id)
        # return redirect(run_url)

    # # @event_logger.log_this
    @expose("/web/<service_pipeline_id>", methods=["GET"])
    def web(self, service_pipeline_id):
        service_pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first()

        # service_pipeline.dag_json = service_pipeline.fix_dag_json()
        # service_pipeline.expand = json.dumps(service_pipeline.fix_position(), indent=4, ensure_ascii=False)

        db.session.commit()
        print(service_pipeline_id)
        data = {
            "url": '/static/appbuilder/vison/index.html?pipeline_id=%s' % service_pipeline_id  # 前后端集成完毕，这里需要修改掉
        }
        # 返回模板
        return self.render_template('link.html', data=data)

    # # @event_logger.log_this
    @expose("/web/log/<service_pipeline_id>", methods=["GET"])
    def web_log(self, service_pipeline_id):
        pass

    # 链路跟踪
    @expose("/web/monitoring/<service_pipeline_id>", methods=["GET"])
    def web_monitoring(self, service_pipeline_id):
        service_pipeline = db.session.query(Service_Pipeline).filter_by(id=int(service_pipeline_id)).first()
        if service_pipeline.run_id:
            url = "http://"+service_pipeline.project.cluster.get('HOST', request.host)+conf.get('GRAFANA_TASK_PATH')+ service_pipeline.name
            return redirect(url)
        else:
            flash('no running instance', 'warning')
            return redirect('/service_pipeline_modelview/web/%s' % service_pipeline.id)

    # # @event_logger.log_this
    @expose("/web/pod/<service_pipeline_id>", methods=["GET"])
    def web_pod(self, service_pipeline_id):
        service_pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first()
        data = {
            "url": "http://" + service_pipeline.project.cluster.get('HOST', request.host) + conf.get('K8S_DASHBOARD_CLUSTER','/k8s/dashboard/cluster/') + '#/search?namespace=%s&q=%s' % (conf.get('SERVICE_PIPELINE_NAMESPACE'), service_pipeline.name.replace('_', '-')),
            "target":"div.kd-chrome-container.kd-bg-background",
            "delay":500,
            "loading": True
        }
        # 返回模板
        if service_pipeline.project.cluster['NAME'] == conf.get('ENVIRONMENT'):
            return self.render_template('link.html', data=data)
        else:
            return self.render_template('external_link.html', data=data)

    @expose('/clear/<service_id>', methods=['POST', "GET"])
    def clear(self, service_pipeline_id):
        service_pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first()

        from myapp.utils.py.py_k8s import K8s
        k8s_client = K8s(service_pipeline.project.cluster.get('KUBECONFIG', ''))
        namespace = conf.get('SERVICE_PIPELINE_NAMESPACE')
        k8s_client.delete_deployment(namespace=namespace, name=service_pipeline.name)

        flash('服务清理完成', category='success')
        return redirect('/service_pipeline_modelview/list/')

    @expose("/config/<service_pipeline_id>", methods=("GET", 'POST'))
    def pipeline_config(self, service_pipeline_id):
        print(service_pipeline_id)
        pipeline = db.session.query(Service_Pipeline).filter_by(id=service_pipeline_id).first()
        if not pipeline:
            return jsonify({
                "status": 1,
                "message": "服务流不存在",
                "result": {}
            })
        if request.method.lower() == 'post':
            data = request.get_json()
            request_config = data.get('config', {})
            request_dag = data.get('dag_json', {})
            if request_config:
                pipeline.config = json.dumps(request_config, indent=4, ensure_ascii=False)
            if request_dag:
                pipeline.dag_json = json.dumps(request_dag, indent=4, ensure_ascii=False)
            db.session.commit()
        config = {
            "id": pipeline.id,
            "name": pipeline.name,
            "label": pipeline.describe,
            "project": pipeline.project.describe,
            "pipeline_ui_config": {
                "alert": {
                    "alert_user": {
                        "type": "str",
                        "item_type": "str",
                        "label": "报警用户",
                        "require": 1,
                        "choice": [],
                        "range": "",
                        "default": "",
                        "placeholder": "报警用户名，逗号分隔",
                        "describe": "报警用户，逗号分隔",
                        "editable": 1,
                        "condition": "",
                        "sub_args": {}
                    }
                }
            },
            "pipeline_jump_button": [
                {
                    "name": "资源查看",
                    "action_url": "",
                    "icon_svg": '<svg t="1644980982636" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2611" width="128" height="128"><path d="M913.937279 113.328092c-32.94432-32.946366-76.898391-51.089585-123.763768-51.089585s-90.819448 18.143219-123.763768 51.089585L416.737356 362.999454c-32.946366 32.94432-51.089585 76.898391-51.089585 123.763768s18.143219 90.819448 51.087539 123.763768c25.406646 25.40767 57.58451 42.144866 93.053326 48.403406 1.76418 0.312108 3.51915 0.463558 5.249561 0.463558 14.288424 0 26.951839-10.244318 29.519314-24.802896 2.879584-16.322757-8.016581-31.889291-24.339338-34.768875-23.278169-4.106528-44.38386-15.081487-61.039191-31.736818-21.61018-21.61018-33.509185-50.489928-33.509185-81.322144s11.899004-59.711963 33.509185-81.322144l15.864316-15.864316c-0.267083 1.121544-0.478907 2.267647-0.6191 3.440355-1.955538 16.45988 9.800203 31.386848 26.260084 33.344432 25.863041 3.072989 49.213865 14.378475 67.527976 32.692586 21.608134 21.608134 33.509185 50.489928 33.509185 81.322144s-11.901051 59.71401-33.509185 81.322144L318.53987 871.368764c-21.61018 21.61018-50.489928 33.511231-81.322144 33.511231-30.832216 0-59.711963-11.901051-81.322144-33.511231-21.61018-21.61018-33.509185-50.489928-33.509185-81.322144s11.899004-59.711963 33.509185-81.322144l169.43597-169.438017c11.720949-11.718903 11.720949-30.722722 0-42.441625-11.718903-11.718903-30.722722-11.718903-42.441625 0L113.452935 666.282852c-32.946366 32.94432-51.089585 76.898391-51.089585 123.763768 0 46.865377 18.143219 90.819448 51.089585 123.763768 32.94432 32.946366 76.898391 51.091632 123.763768 51.091632s90.819448-18.145266 123.763768-51.091632l249.673409-249.671363c32.946366-32.94432 51.089585-76.898391 51.089585-123.763768-0.002047-46.865377-18.145266-90.819448-51.089585-123.763768-27.5341-27.536146-64.073294-45.240367-102.885252-49.854455-3.618411-0.428765-7.161097-0.196475-10.508331 0.601704l211.589023-211.589023c21.61018-21.61018 50.489928-33.509185 81.322144-33.509185s59.711963 11.899004 81.322144 33.509185c21.61018 21.61018 33.509185 50.489928 33.509185 81.322144s-11.899004 59.711963-33.509185 81.322144l-150.180418 150.182464c-11.720949 11.718903-11.720949 30.722722 0 42.441625 11.718903 11.718903 30.722722 11.718903 42.441625 0l150.180418-150.182464c32.946366-32.94432 51.089585-76.898391 51.089585-123.763768C965.026864 190.226482 946.882622 146.272411 913.937279 113.328092z" p-id="2612" fill="#225ed2"></path></svg>'
                }
            ],
            "pipeline_run_button": [
            ],
            "task_jump_button": [],
            "dag_json": json.loads(pipeline.dag_json),
            "config": json.loads(pipeline.config),
            "message": "success",
            "status": 0
        }
        return jsonify(config)

    @expose("/template/list/")
    def template_list(self):

        all_template = {
            "message": "success",
            "templte_common_ui_config": {
            },
            "template_group_order": ["入口", "逻辑节点", "功能节点"],
            "templte_list": {
                "入口": [
                    {
                        "template_name": "kafka",
                        "template_id": 1,
                        "templte_ui_config": {
                            "shell": {
                                "topic": {
                                    "type": "str",
                                    "item_type": "str",
                                    "label": "topic",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "predict",
                                    "placeholder": "",
                                    "describe": "kafka topic",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "consumer_num": {
                                    "type": "str",
                                    "item_type": "str",
                                    "label": "消费者数目",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "4",
                                    "placeholder": "",
                                    "describe": "消费者数目",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "bootstrap_servers": {
                                    "type": "str",
                                    "item_type": "str",
                                    "label": "地址",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "127.0.0.1:9092",
                                    "placeholder": "",
                                    "describe": "xx.xx.xx.xx:9092,xx.xx.xx.xx:9092",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "group": {
                                    "type": "str",
                                    "item_type": "str",
                                    "label": "分组",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "predict",
                                    "placeholder": "",
                                    "describe": "消费者分组",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                }
                            }
                        },
                        "username": g.user.username,
                        "changed_on": datetime.datetime.now(),
                        "created_on": datetime.datetime.now(),
                        "label": "kafka",
                        "describe": "消费kafka数据",
                        "help_url": "",
                        "pass_through": {
                            # 无论什么内容  通过task的字段透传回来
                        }
                    }
                ],
                "逻辑节点": [
                    {
                        "template_name": "switch",
                        "template_id": 2,
                        "templte_ui_config": {
                            "shell": {
                                "case": {
                                    "type": "text",
                                    "item_type": "str",
                                    "label": "表达式",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "int(input['node2'])<3:node4,node5:'3'\ndefault:node6:'0'",
                                    "placeholder": "",
                                    "describe": "条件:下游节点:输出     其中input为节点输入",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                }
                            }
                        },
                        "username": g.user.username,
                        "changed_on": datetime.datetime.now(),
                        "created_on": datetime.datetime.now(),
                        "label": "switch-case逻辑节点",
                        "describe": "控制数据的流量",
                        "help_url": "",
                        "pass_through": {
                            # 无论什么内容  通过task的字段透传回来
                        }

                    },
                ],
                "功能节点": [
                    {
                        "template_name": "http",
                        "template_id": 3,
                        "templte_ui_config": {
                            "shell": {
                                "method": {
                                    "type": "choice",
                                    "item_type": "str",
                                    "label": "请求方式",
                                    "require": 1,
                                    "choice": ["GET", "POST"],
                                    "range": "",
                                    "default": "POST",
                                    "placeholder": "",
                                    "describe": "请求方式",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "url": {
                                    "type": "str",
                                    "item_type": "str",
                                    "label": "请求地址",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "http://127.0.0.1:8080/api",
                                    "placeholder": "",
                                    "describe": "请求地址",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "header": {
                                    "type": "text",
                                    "item_type": "str",
                                    "label": "请求头",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "{}",
                                    "placeholder": "",
                                    "describe": "请求头",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "timeout": {
                                    "type": "int",
                                    "item_type": "str",
                                    "label": "请求超时",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "300",
                                    "placeholder": "",
                                    "describe": "请求超时",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                                "date": {
                                    "type": "text",
                                    "item_type": "str",
                                    "label": "请求内容",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "{}",
                                    "placeholder": "",
                                    "describe": "请求内容",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                }
                            }
                        },
                        "username": g.user.username,
                        "changed_on": datetime.datetime.now(),
                        "created_on": datetime.datetime.now(),
                        "label": "http请求",
                        "describe": "http请求",
                        "help_url": "",
                        "pass_through": {
                            # 无论什么内容  通过task的字段透传回来
                        }
                    },
                    {
                        "template_name": "自定义方法",
                        "template_id": 4,
                        "templte_ui_config": {
                            "shell": {
                                "sdk_path": {
                                    "type": "str",
                                    "item_type": "str",
                                    "label": "函数文件地址",
                                    "require": 1,
                                    "choice": [],
                                    "range": "",
                                    "default": "",
                                    "placeholder": "",
                                    "describe": "函数文件地址，文件名和python类型要相同",
                                    "editable": 1,
                                    "condition": "",
                                    "sub_args": {}
                                },
                            }
                        },
                        "username": g.user.username,
                        "changed_on": datetime.datetime.now(),
                        "created_on": datetime.datetime.now(),
                        "label": "http请求",
                        "describe": "http请求",
                        "help_url": "",
                        "pass_through": {
                            # 无论什么内容  通过task的字段透传回来
                        }

                    }
                ],

            },
            "status": 0
        }
        index = 1
        for group in all_template['templte_list']:
            for template in all_template['templte_list'][group]:
                template['template_id'] = index
                template['changed_on'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                template['created_on'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                template['username'] = g.user.username,
                index += 1

        return jsonify(all_template)


class Service_Pipeline_ModelView(Service_Pipeline_ModelView_Base, MyappModelView, DeleteMixin):
    datamodel = SQLAInterface(Service_Pipeline)
    # base_order = ("changed_on", "desc")
    # order_columns = ['changed_on']


appbuilder.add_view_no_menu(Service_Pipeline_ModelView)


# 添加api
class Service_Pipeline_ModelView_Api(Service_Pipeline_ModelView_Base, MyappModelRestApi):
    datamodel = SQLAInterface(Service_Pipeline)
    route_base = '/service_pipeline_modelview/api'
    show_columns = ['project', 'name', 'describe', 'namespace', 'node_selector', 'image_pull_policy', 'env', 'dag_json',
                    'run_id', 'created_by', 'changed_by', 'created_on', 'changed_on', 'expand']
    list_columns = ['project', 'service_pipeline_url', 'creator', 'modified', 'operate_html']
    add_columns = ['project', 'name', 'describe', 'namespace', 'node_selector', 'image_pull_policy', 'dag_json', 'env',
                   'expand']
    edit_columns = add_columns


appbuilder.add_api(Service_Pipeline_ModelView_Api)