mirror/cube-studio
2
0
Fork 0
mirror of https://github.com/tencentmusic/cube-studio.git synced 2024-11-21 01:16:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix dimension

Browse Source
This commit is contained in:
cdllp2 2022-09-20 20:20:41 +08:00
parent 787892490e
commit 1663e9e545
4 changed files with 76 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
myapp/models/model_dimension.py
View File

@ -68,8 +68,8 @@ class Dimension_table(Model,ImportMixin,MyappModelBase):
@property
def operate_html(self):
url=f'''
<a target=_blank href="/dimension_table_modelview/api/create_external_table/%s">创建远程表</a> | <a target=_blank href="/dimension_table_modelview/api/download/%s">下载</a> | <a target=_blank href="/dimension_table_modelview/api/csv/%s">上传模板</a> | <a target=_blank href="/dimension_table_modelview/api/external/%s">建外表示例</a> | <a href="/dimension_table_modelview/api/clear/%s">清空表记录</a>
'''%(self.id,self.id,self.id,self.id,self.id)
<a target=_blank href="/dimension_table_modelview/api/create_external_table/%s">创建远程表</a> | <a target=_blank href="/dimension_table_modelview/api/external/%s">建外表示例</a> | <a href="/dimension_table_modelview/api/clear/%s">清空表记录</a>
'''%(self.id,self.id,self.id)
return Markup(url)

7
myapp/views/baseApi.py
View File

@ -1352,15 +1352,16 @@ class MyappModelRestApi(ModelRestApi):
uri = url.make_url(sqllchemy_uri)
sql_engine = create_engine(uri)
table_name = self.datamodel.obj.__tablename__
sql = 'select `%s` from %s' % ('`,`'.join(self.show_columns), table_name)
# print(sql)
# sql = 'select `%s` from %s' % ('`,`'.join(self.show_columns), table_name)
sql = 'select * from %s' % (table_name)
print(sql)
results = pandas.read_sql_query(sql, sql_engine)
file_path = '%s.csv' % table_name
csv_file = os.path.abspath(file_path)
if os.path.exists(csv_file):
os.remove(csv_file)
results.to_csv(csv_file, index=False, sep=",") # index 是第几行的表示
results.to_csv(csv_file, index=False, sep=",",encoding='utf-8-sig') # index 是第几行的表示
response = self.csv_response(csv_file, file_name=table_name)
return response

15
myapp/views/home.py
View File

@ -59,6 +59,21 @@ class Myapp(BaseMyappView):
# 返回模板
return self.render_template('home.html')
@expose('/navbar_right')
def navbar_right(self):
return [
{
"text":"帮助文档",
"icon":'<svg t="1663658395713" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4531" width="64" height="64"><path d="M808.35715 213.332736c-164.707272-164.709319-432.683537-164.709319-597.333504 0-164.709319 164.649967-164.709319 432.624185 0 597.333504 164.649967 164.709319 432.626231 164.709319 597.333504 0C973.067492 645.956921 973.067492 377.982704 808.35715 213.332736zM765.48983 767.767198c-141.075039 141.077086-370.609783 141.077086-511.627517 0-141.078109-141.017734-141.078109-370.520755 0-511.596817 141.017734-141.017734 370.552477-141.077086 511.627517 0C906.506541 397.188114 906.506541 626.750487 765.48983 767.767198z" p-id="4532" fill="#3273f1"></path><path d="M500.686838 686.040848c-20.432355 0.061398-37.063127 16.692171-37.063127 37.095873 0 20.458961 16.630772 37.090756 37.092803 37.090756 20.432355 0 37.066197-16.631796 37.066197-37.151132C537.781688 702.672644 521.147846 686.040848 500.686838 686.040848z" p-id="4533" fill="#3273f1"></path><path d="M505.832022 265.144776c-72.362075 0-136.290059 41.640376-158.933779 103.683431-5.26491 14.508435-9.813506 38.620599-5.26491 58.753125 3.048429 13.81975 17.709337 22.974247 31.320333 19.863397 14.117532-3.171226 23.0029-17.23043 19.832697-31.290657-1.556449-7.210215-0.060375-20.132526 3.350304-29.435403 12.533454-34.342156 51.27378-69.043493 109.60735-69.043493 68.681242 0 116.635417 35.091216 116.635417 85.375459 0 37.572734-23.302729 51.1561-64.525596 72.034617-37.303604 18.847254-83.761706 42.241057-83.761706 101.289917L474.092134 629.981065c0 14.478759 11.784394 26.2652 26.204825 26.2652 14.449084 0 26.235524-11.786441 26.235524-26.2652l0-53.606919c0.060375-25.605168 18.936281-36.317137 55.042617-54.504358 41.6107-20.968567 93.421716-47.025013 93.421716-118.820176C674.997839 321.862545 605.418135 265.144776 505.832022 265.144776z" p-id="4534" fill="#3273f1"></path></svg>',
"link":"https://github.com/tencentmusic/cube-studio/wiki"
},
{
"text":'',
"icon":'<svg t="1663658292626" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2369" width="64" height="64"><path d="M511.6 76.3C264.3 76.2 64 276.4 64 523.5 64 718.9 189.3 885 363.8 946c23.5 5.9 19.9-10.8 19.9-22.2v-77.5c-135.7 15.9-141.2-73.9-150.3-88.9C215 726 171.5 718 184.5 703c30.9-15.9 62.4 4 98.9 57.9 26.4 39.1 77.9 32.5 104 26 5.7-23.5 17.9-44.5 34.7-60.8-140.6-25.2-199.2-111-199.2-213 0-49.5 16.3-95 48.3-131.7-20.4-60.5 1.9-112.3 4.9-120 58.1-5.2 118.5 41.6 123.2 45.3 33-8.9 70.7-13.6 112.9-13.6 42.4 0 80.2 4.9 113.5 13.9 11.3-8.6 67.3-48.8 121.3-43.9 2.9 7.7 24.7 58.3 5.5 118 32.4 36.8 48.9 82.7 48.9 132.3 0 102.2-59 188.1-200 212.9 23.5 23.2 38.1 55.4 38.1 91v112.5c0.8 9 0 17.9 15 17.9 177.1-59.7 304.6-227 304.6-424.1 0-247.2-200.4-447.3-447.5-447.3z" p-id="2370" fill="#3273f1"></path></svg>',
"link":"https://github.com/tencentmusic/cube-studio"
}
]
# 返回模板
@expose('/menu')
def menu(self):

105
myapp/views/view_dimension.py
View File

@ -110,7 +110,7 @@ Metadata_column_fields = {
description='列类型',
widget=Select2Widget(),
default='text',
choices=[['int', 'int'], ['text', 'text'],['date', 'date']],
choices=[['int', 'int'], ['text', 'text'],['date', 'date'],['double','double']],
validators=[DataRequired()]
),
"unique": BooleanField(
@ -333,52 +333,11 @@ class Dimension_table_ModelView_Api(MyappModelRestApi):
import sqlalchemy.engine.url as url
uri = url.make_url(dim.sqllchemy_uri)
sql_engine = create_engine(uri)
columns = list(json.loads(dim.columns).values())
cols = [col['name'] for col in columns if not col.get('primary_key',False)]
sql = 'select %s from %s' % (','.join(cols), dim.table_name)
sql = 'select * from %s' % (dim.table_name,)
results = pandas.read_sql_query(sql, sql_engine)
return results.to_dict()
@expose("/csv/<dim_id>", methods=["GET"])
# @pysnooper.snoop()
def csv(self,dim_id):
dim = db.session.query(Dimension_table).filter_by(id=int(dim_id)).first()
cols = json.loads(dim.columns)
for col_name in copy.deepcopy(cols):
if cols[col_name].get('primary_key',False):
del cols[col_name]
demostr=','.join(list(cols.keys()))+"\n"+','.join(['xx' for x in list(cols.keys())])
csv_file='%s.csv'%dim_id
file = open(csv_file,mode='w',encoding='utf-8-sig')
file.writelines(demostr)
file.close()
csv_file = os.path.abspath(csv_file)
response = self.csv_response(csv_file,file_name=dim.table_name)
return response
# return ','.join(list(cols.keys()))+"<br>"+','.join(['xx' for x in list(cols.keys())])
@expose("/download/<dim_id>", methods=["GET"])
# @pysnooper.snoop()
def download(self,dim_id):
import pandas
dim = db.session.query(Dimension_table).filter_by(id=int(dim_id)).first()
import sqlalchemy.engine.url as url
uri = url.make_url(dim.sqllchemy_uri)
sql_engine = create_engine(uri)
columns = list(json.loads(dim.columns).values())
cols = [col['name'] for col in columns if not col.get('primary_key', False)]
sql = 'select %s from %s' % (','.join(cols), dim.table_name)
results = pandas.read_sql_query(sql, sql_engine)
file_path = '%s.csv' % dim.table_name
csv_file = os.path.abspath(file_path)
if os.path.exists(csv_file):
os.remove(csv_file)
results.to_csv(csv_file, index=False, sep=",") # index 是第几行的表示
response = self.csv_response(csv_file, file_name=dim.table_name)
return response
@expose("/external/<dim_id>", methods=["GET"])
def external(self,dim_id):
ddl_sql = ddl_hive_external_table(dim_id)
@ -412,9 +371,9 @@ class Dimension_table_ModelView_Api(MyappModelRestApi):
item = db.session.query(Dimension_table).filter_by(id=int(dim_id)).first()
sqllchemy_uri = item.sqllchemy_uri
if sqllchemy_uri:
# 创建数据库的sql(如果数据库存在就不创建，防止异常)
if 'postgresql' in item.sqllchemy_uri:
# 创建pg表
import sqlalchemy.engine.url as url
uri = url.make_url(sqllchemy_uri)
@ -496,7 +455,11 @@ class Dimension_table_ModelView_Api(MyappModelRestApi):
if exist_columns:
col = json.loads(item.columns)
for column_name in col:
col_type = 'INT' if col[column_name].get('column_type','text').upper() == 'INT' else 'varchar(2000)'
col_type = 'varchar(2000)'
if col[column_name].get('column_type', 'text').upper() == 'INT':
col_type = 'INT'
if col[column_name].get('column_type', 'text').upper() in ['DOUBLE','FLOAT']:
col_type = 'DOUBLE'
if column_name not in exist_columns:
try:
sql = 'ALTER TABLE %s ADD %s %s;'%(item.table_name,column_name,col_type)
@ -523,7 +486,7 @@ class Dimension_table_ModelView_Api(MyappModelRestApi):
'''.format(
table_name=item.table_name,
columns_sql='\n'.join(
[" %s %s %s %s," % (col_name, 'BIGINT' if cols[col_name].get('column_type','text').upper() == 'INT' else 'varchar(2000)',
[" %s %s %s %s," % (col_name, 'BIGINT' if cols[col_name].get('column_type','text').upper() == 'INT' else 'DOUBLE' if cols[col_name].get('column_type','text').upper() in ['DOUBLE','FLOAT'] else 'varchar(2000)',
'' if int(cols[col_name].get('nullable', True)) else 'NOT NULL',
'' if not int(cols[col_name].get('unique', False)) else 'UNIQUE') for
col_name in cols if col_name not in ['id',]]
@ -614,6 +577,11 @@ class Dimension_remote_table_ModelView_Api(MyappModelRestApi):
"type": "ellip1",
"width": 100
}
if column_type == 'double':
cols_width[column_type] ={
"type": "ellip1",
"width": 100
}
if column_type == 'date':
cols_width[column_type] ={
"type": "ellip1",
@ -688,6 +656,30 @@ class Dimension_remote_table_ModelView_Api(MyappModelRestApi):
url = '/dimension_remote_table_modelview/%s/api/' % dim_id
print(url)
# 预处理一下
# @pysnooper.snoop(watch_explode=('item'))
def pre_add(self,item):
# 浮点转型
for key in self.cols:
if key in ['id','rowid']:
continue
if self.cols[key].get('column_type', 'text') == 'int':
setattr(item,key,int(getattr(item,key)) if getattr(item,key) else None)
if self.cols[key].get('column_type', 'text') == 'double':
setattr(item,key,float(getattr(item,key)) if getattr(item,key) else None)
def pre_update(self,item):
# 浮点转型
for key in self.cols:
if key in ['id','rowid']:
continue
if self.cols[key].get('column_type', 'text') == 'int':
setattr(item,key,int(getattr(item,key)) if getattr(item,key) else None)
if self.cols[key].get('column_type', 'text') == 'double':
setattr(item,key,float(getattr(item,key)) if getattr(item,key) else None)
def get_primary_key(cols):
for name in cols:
if cols[name].get('primary_key',False):
@ -711,6 +703,7 @@ class Dimension_remote_table_ModelView_Api(MyappModelRestApi):
header = None
result = []
cols = json.loads(dim.columns)
error_message = []
for line in csv_reader:
if not header:
header = line
@ -737,12 +730,17 @@ class Dimension_remote_table_ModelView_Api(MyappModelRestApi):
data = dict(zip(header, line))
try:
# 把整型做一下转换，因为文件离线全部识别为字符串
for key in data:
for key in copy.deepcopy(data):
try:
data[key]=int(data[key]) if cols.get(key,{}).get('column_type','text')=='int' else str(data[key])
if cols.get(key,{}).get('column_type','text')=='int':
data[key]=int(data[key])
elif cols.get(key,{}).get('column_type','text')=='double':
data[key]=float(data[key]) if data[key] else None
else:
data[key]=str(data[key])
except Exception as e:
data[key] = None
@ -757,9 +755,12 @@ class Dimension_remote_table_ModelView_Api(MyappModelRestApi):
except Exception as e:
db.session.rollback()
print(e)
error_message.append(str(e))
result.append('fail')
flash('成功导入%s行，失败导入%s行' % (len([x for x in result if x == 'success']), len([x for x in result if x == 'fail'])), 'success')
error_message='<br>'.join(error_message)
flash('上传失败%s'%error_message,'error')
back = {
"status": 0,
"result": result,
@ -802,13 +803,17 @@ class Dimension_remote_table_ModelView_Api(MyappModelRestApi):
search_columns=search_columns,
order_columns=order_columns,
label_title = dim.label,
base_permissions = ['can_list','can_add','can_delete','can_edit','can_show'],
pre_add=pre_add,
pre_update=pre_update,
upload=upload,
muldelete=muldelete,
dim_id=dim_id,
import_data=True,
download_data=True,
cols_width=cols_width,
base_order=(get_primary_key(columns), "desc") if get_primary_key(columns) else None
base_order=(get_primary_key(columns), "desc") if get_primary_key(columns) else None,
cols = columns
)
)
view_instance = view_class()