解决SQL查询bigint前端显示丢失精度的问题

解决SQL查询结果html显示未转义的问题优化detail详情显示，提升审核体验
gaoxianglong · May 5, 2018 · d6e4b5b · d6e4b5b
1 parent d66e7cc
commit d6e4b5b
Show file tree

Hide file tree

Showing 13 changed files with 118 additions and 35 deletions.
diff --git a/archer/settings.py b/archer/settings.py
@@ -245,10 +245,10 @@
 # 是否开启SQL查询功能，关闭会隐藏菜单和相关功能
 QUERY = False
 
-# 当inception语法树打印失败时在线查询的结果控制，
+# 当inception语法树打印失败时在线查询的结果控制，建议修改inception变量inception_enable_select_star=OFF，否则select * 会报错
 # True是开启校验，失败不允许继续执行并返回错，
 # False是关闭校验，继续执行，关闭校验会导致解析失败的查询表权限验证和脱敏功能失效
-CHECK_QUERY_ON_OFF = True
+CHECK_QUERY_ON_OFF = False
 
 # 是否开启动态脱敏查询，采取正则遍历处理结果集的方式，会影响部分查询效率
 DATA_MASKING_ON_OFF = False

diff --git a/requirements.txt b/requirements.txt
@@ -9,7 +9,7 @@ PyMySQL==0.7.11
 PyYAML==3.12
 requests==2.18.4
 shellescape==3.4.1
-simplejson==3.13.2
+simplejson==3.14.0
 urllib3==1.22
 django-admin-bootstrapped==2.5.7
 django-apscheduler==0.2.8

diff --git a/sql/data_masking.py b/sql/data_masking.py
@@ -1,7 +1,7 @@
 # -*- coding:utf-8 -*-
 from .inception import InceptionDao
 from .models import DataMaskingRules, DataMaskingColumns
-import json
+import simplejson as json
 import re
 
 inceptionDao = InceptionDao()

diff --git a/sql/extend_json_encoder.py b/sql/extend_json_encoder.py
@@ -0,0 +1,65 @@
+# -*- coding: UTF-8 -*-
+import simplejson as json
+
+from datetime import datetime, date
+from decimal import Decimal
+from functools import singledispatch
+
+
+class MyClass:
+    def __init__(self, value):
+        self._value = value
+
+    def get_value(self):
+        return self._value
+
+
+# 创建非内置类型的实例
+mc = MyClass('i am class MyClass ')
+dm = Decimal('11.11')
+dt = datetime.now()
+dat = date.today()
+
+
+@singledispatch
+def convert(o):
+    raise TypeError('can not convert type')
+
+
+@convert.register(datetime)
+def _(o):
+    return o.strftime('%Y-%m-%d %H:%M:%S')
+
+
+@convert.register(date)
+def _(o):
+    return o.strftime('%Y-%m-%d')
+
+
+# @convert.register(Decimal)
+# def _(o):
+#     return float(o)
+
+
+@convert.register(MyClass)
+def _(o):
+    return o.get_value()
+
+
+class ExtendJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        try:
+            return convert(obj)
+        except TypeError:
+            return super(ExtendJSONEncoder, self).default(obj)
+
+
+data = {
+    'mc': mc,
+    'dm': dm,
+    'dt': dt,
+    'dat': dat,
+    'bigint': 988983860501598208
+}
+
+#print(json.dumps(data, cls=ExtendJSONEncoder, bigint_as_string=True))
diff --git a/sql/inception.py b/sql/inception.py
@@ -1,7 +1,7 @@
 #-*-coding: utf-8-*-
 
 import re
-import json
+import simplejson as json
 import MySQLdb
 from django.conf import settings
 from django.db import connection

diff --git a/sql/permission.py b/sql/permission.py
@@ -1,5 +1,5 @@
 # -*- coding: UTF-8 -*-
-import json
+import simplejson as json
 from django.shortcuts import render
 from django.http import HttpResponse
 from .models import users

diff --git a/sql/query.py b/sql/query.py
@@ -14,6 +14,7 @@
 import datetime
 import time
 
+from sql.extend_json_encoder import ExtendJSONEncoder
 from .aes_decryptor import Prpcrypt
 from .sendmail import MailSender
 from .dao import Dao
@@ -37,17 +38,6 @@
 mailSenderOb = MailSender()
 
 
-# 处理查询结果的时间格式
-class DateEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, datetime.datetime):
-            return obj.strftime('%Y-%m-%d %H:%M:%S')
-        elif isinstance(obj, date):
-            return obj.strftime("%Y-%m-%d")
-        else:
-            return json.JSONEncoder.default(self, obj)
-
-
 # 查询权限申请用于工作流审核回调
 def query_audit_call_back(workflow_id, workflow_status):
     # 更新业务表状态
@@ -673,7 +663,7 @@ def query(request):
             query_log.save()
 
     # 返回查询结果
-    return HttpResponse(json.dumps(finalResult, cls=DateEncoder), content_type='application/json')
+    return HttpResponse(json.dumps(finalResult, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json')
 
 
 # 获取sql查询记录
@@ -759,7 +749,7 @@ def explain(request):
     finalResult['data'] = sql_result
 
     # 返回查询结果
-    return HttpResponse(json.dumps(finalResult, cls=DateEncoder), content_type='application/json')
+    return HttpResponse(json.dumps(finalResult, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json')
 
 
 # 获取SQL慢日志统计
@@ -859,14 +849,11 @@ def slowquery_review(request):
                 ReturnTotalRowCounts=Sum('slowqueryhistory__rows_sent_sum'),  # 返回总行数
             ).count()
         # QuerySet 序列化
-        SQLSlowLog = []
-        for SlowLog in slowsql_obj:
-            SlowLog['SQLId'] = str(SlowLog['SQLId'])
-            SQLSlowLog.append(SlowLog)
+        SQLSlowLog = [SlowLog for SlowLog in slowsql_obj]
         result = {"total": slowsql_obj_count, "rows": SQLSlowLog}
 
     # 返回查询结果
-    return HttpResponse(json.dumps(result, cls=DateEncoder), content_type='application/json')
+    return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json')
 
 
 # 获取SQL慢日志明细
@@ -972,4 +959,4 @@ def slowquery_review_history(request):
         result = {"total": slowsql_obj_count, "rows": SQLSlowRecord}
 
         # 返回查询结果
-    return HttpResponse(json.dumps(result, cls=DateEncoder), content_type='application/json')
+    return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json')
diff --git a/sql/sqlreview.py b/sql/sqlreview.py
@@ -1,5 +1,5 @@
 # -*- coding: UTF-8 -*-
-import json
+import simplejson as json
 
 import time
 from threading import Thread

diff --git a/sql/static/detail.html b/sql/static/detail.html
@@ -386,7 +386,23 @@ <h4 class="modal-title text-danger">定时执行SQL</h4>
                 uniqueId: "id",                     //每一行的唯一标识，一般为主键列
                 showToggle: false,                   //是否显示详细视图和列表视图的切换按钮
                 cardView: false,                    //是否显示详细视图
-                detailView: false,                   //是否显示父子表
+                detailView: true,                  //是否显示父子表
+                //格式化详情
+                detailFormatter: function (index, row) {
+                    var html = [];
+                    $.each(row, function (key, value) {
+                        if (key === 'SQL') {
+                            var sql = value;
+                            //替换所有的换行符
+                            sql = sql.replace(/\r\n/g, "<br>");
+                            sql = sql.replace(/\n/g, "<br>");
+                            //替换所有的空格
+                            sql = sql.replace(/\s/g, "&nbsp;");
+                            html.push('<span>' + sql + '</span>');
+                        }
+                    });
+                    return html.join('');
+                },
                 locale: 'zh-CN',                    //本地化
                 data:{{ rows|safe }},
                 columns: [{
@@ -397,7 +413,12 @@ <h4 class="modal-title text-danger">定时执行SQL</h4>
                     title: 'SQL内容',
                     field: 'SQL',
                     formatter: function (value, row, index) {
-                        return value.replace(/\n/g, '<br>');
+                        if (value.length > 80) {
+                            return value.substr(0, 80) + '...';
+                        }
+                        else {
+                            return value
+                        }
                     },
                     sortable: true
                 }, {

diff --git a/sql/static/queryapplylist.html b/sql/static/queryapplylist.html
@@ -340,6 +340,7 @@ <h4 class="modal-title" id="myModalLabel">申请数据库查询权限</h4>
             else if ($("#priv_type").val() === '2') {
                 $("#table_div").show();
                 $("#db_name_div").show();
+                $("#db_name_multiple_div").hide();
             }
         });
 

diff --git a/sql/static/sqlquery.html b/sql/static/sqlquery.html
@@ -351,7 +351,7 @@
             }
             //校验SQL
             sqlContent = $.trim(sqlContent);
-            if (sqlContent.toLowerCase().match(/^select/) || sqlContent.toLowerCase().match(/^show.*create/) || sqlContent.toLowerCase().match(/^explain/)) {
+            if (sqlContent.toLowerCase().match(/^select/) || sqlContent.toLowerCase().match(/^show.*create.*table/) || sqlContent.toLowerCase().match(/^explain/)) {
                 $.ajax({
                     type: "post",
                     url: "/query/",
@@ -423,7 +423,15 @@
                             else if (result['column_list']) {
                                 //异步获取要动态生成的列
                                 $.each(result['column_list'], function (i, column) {
-                                    columns.push({"field": i, "title": column, "sortable": true});
+                                    columns.push({
+                                        "field": i,
+                                        "title": column,
+                                        "sortable": true,
+                                        "formatter": function (value, row, index) {
+                                            //return value;
+                                            return $('<div/>').text(value).html();
+                                        }
+                                    });
                                 });
                                 /*//插入选择框
                                 columns.unshift({
@@ -439,7 +447,7 @@
                                                 title: 'Create Table',
                                                 field: 1,
                                                 formatter: function (value, row, index) {
-                                                    var sql = value;
+                                                    var sql = window.sqlFormatter.format(value);
                                                     //替换所有的换行符
                                                     sql = sql.replace(/\r\n/g, "<br>");
                                                     sql = sql.replace(/\n/g, "<br>");

diff --git a/sql/views.py b/sql/views.py
@@ -1,7 +1,7 @@
 # -*- coding: UTF-8 -*-
 import datetime
 import re
-import json
+import simplejson as json
 from threading import Thread
 from collections import OrderedDict
 

diff --git a/sql/views_ajax.py b/sql/views_ajax.py
@@ -1,7 +1,7 @@
 # -*- coding: UTF-8 -*- 
 
 import re
-import json
+import simplejson as json
 import datetime
 import multiprocessing
 
@@ -29,7 +29,8 @@
 from sql.sendmail import MailSender
 import logging
 from .workflow import Workflow
-from .query import query_audit_call_back, DateEncoder
+from .query import query_audit_call_back
+from .extend_json_encoder import ExtendJSONEncoder
 
 logger = logging.getLogger('default')
 mailSender = MailSender()
@@ -206,7 +207,7 @@ def sqlworkflow(request):
 
     result = {"total": listWorkflowCount, "rows": rows}
     # 返回查询结果
-    return HttpResponse(json.dumps(result, cls=DateEncoder), content_type='application/json')
+    return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json')
 
 
 # 提交SQL给inception进行自动审核