Merge branch 'master'

xydoublez · Apr 13, 2018 · 1783293 · 1783293
1 parent a9f0ee9
commit 1783293
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 315 deletions.
diff --git a/README.md b/README.md
@@ -17,12 +17,12 @@ linux : 64位linux操作系统均可
   很多时候DBA并不知道SQL的业务含义，所以人工审核最好由其他研发工程师或研发经理来审核. 这是archer的设计理念.
 * 回滚数据展示<br/>
 * 在线查询<br/>
-  查询权限控制，基于inception解析查询语句，查询权限限制到表级
+  查询权限控制，基于inception解析查询语句，查询权限支持限制到表级<br/>
   查询权限申请、审核和管理，支持审核流程配置<br/>
-  查询结果集限制、查询结果导出、表结构展示、多结果级展示<br/>
+  查询结果集限制、查询结果导出、表结构展示、多结果集展示<br/>
 * 动态脱敏<br/> 
   基于inception解析查询语句，配合脱敏字段配置、脱敏规则(正则表达式)实现动态脱敏<br/>
-* 主库集群配置
+* 主库集群配置<br/>
 * 用户权限配置<br/>
   工程师角色（engineer）与审核角色（review_man）:工程师可以发起SQL上线，在通过了inception自动审核之后，需要由人工审核点击确认才能执行SQL.<br/>
   还有一个特殊的超级管理员即可以上线、审核，又可以登录admin界面进行管理.
@@ -50,16 +50,8 @@ cd Python-3.4.1 <br/>
 ./configure --prefix=/path/to/python3 && make && make install
 或者rpm、yum、binary等其他安装方式
 3. 安装所需相关模块：<br/>
-(1)django：<br/>
-tar -xzvf Django-1.8.17 && cd Django-1.8.17 && python3 setup.py install<br/>
-或者pip3 install Django==1.8.17<br/>
-(2)Crypto:<br/>
-pip3 install Crypto<br/>
-pip3 install pycrypto<br/>
-(3)其他模块:<br/>
 pip3 install -r requirements.txt<br/>
 4. 给python3安装MySQLdb模块:<br/>
-pip3 install pymysql<br/>
 记得确保settings.py里有如下两行：<br/>
 import pymysql<br/>
 pymysql.install_as_MySQLdb()<br/>
@@ -82,13 +74,13 @@ python3 manage.py migrate<br/>
 cd archer && python3 manage.py createsuperuser<br/>
 8. 启动，有两种方式：<br/>
 (1)用django内置runserver启动服务,需要修改debug.sh里的ip和port<br/>
-    cd archer && bash debug.sh<br/>
+cd archer && bash debug.sh<br/>
 (2)用gunicorn启动服务，可以使用pip3 install gunicorn安装并用startup.sh启动，但需要配合nginx处理静态资源. (nginx安装这里不做示范)<br/>
     * gunicorn的安装配置示例:
-        * pip3 install gunicorn
 	    * cat startup.sh
             * ![image](https://github.com/jly8866/archer/raw/master/screenshots/startup.png)<br/>
     * nginx配置示例：
+        * 静态资源地址请指定setting.py里面的STATIC_ROOT配置项地址，一般为/archer/static
         * cat nginx.conf
             * ![image](https://github.com/jly8866/archer/raw/master/screenshots/nginx.png)<br/>
 9. 创建archer系统登录用户：<br/>
@@ -102,8 +94,7 @@ cd archer && python3 manage.py createsuperuser<br/>
 这一步是为了进行sql在线查询，所用到的用户名密码、端口等，建议账号仅开放SELECT权限。<br/>
 12. 配置查询权限审核人：<br/>
 使用浏览器访问http://X.X.X.X:port/admin/sql/workflowauditsetting/ ，点击右侧Add 工作流配置<br/>
-这一步是为了添加查询权限审核人，单人审核格式为：user1，多人审核格式为：user1,user2，请正确配置。<br/>
-
+这一步是为了添加查询权限审核人，单人审核格式为：user1，多级审核格式为：user1,user2，请正确配置。<br/>
 13. 正式访问：<br/>
 以上步骤完毕，就可以使用步骤9创建的用户登录archer系统啦, 首页地址 http://X.X.X.X:port/<br/>
 <br/>
@@ -115,6 +106,23 @@ cd archer && python3 manage.py createsuperuser<br/>
 3. centos需要执行yum install openldap-devel<br/>
 4. settings中以AUTH_LDAP开头的配置，需要根据自己的ldap对应修改<br/>
 
+### 集成SQLAdvisor
+1. 安装SQLAdvisor，[项目地址](https://github.com/Meituan-Dianping/SQLAdvisor)
+2. 修改配置文件SQLADVISOR为程序路径，路径需要完整，如'/opt/SQLAdvisor/sqladvisor/sqladvisor'
+
+### 慢日志管理
+1. 安装percona-toolkit（版本>3.0），以centos为例   
+    yum -y install http://www.percona.com/downloads/percona-release/redhat/0.1-3/percona-release-0.1-3.noarch.rpm 
+    yum -y install percona-toolkit.x86_64 
+2. 使用src/script/mysql_slow_query_review.sql创建慢日志收集表
+3. 将src/script/analysis_slow_query.sh部署到各个监控机器，注意按照说明修改配置信息
+4. 如果有阿里云rds实例，可以在后台数据管理模块添加关联关系，可直接拉取rds慢日志
+
+### 集成阿里云rds管理
+1. 修改配置文件ENABLE_ALIYUN=True
+2. 访问http://X.X.X.X:port/admin/sql/aliyunaccesskey/, 添加aliyun账号的accesskey信息，重新启动服务
+3. 访问http://X.X.X.X:port/admin/sql/aliyunrdsconfig/，添加实例id信息即可实现rds进程管理、慢日志管理
+
 ### admin后台加固，防暴力破解
 * 1.patch目录下，名称为：django_1.8.17_admin_secure_archer.patch
 * 2.使用命令：patch  python/site-packages/django/contrib/auth/views.py django_1.8.17_admin_secure_archer.patch
@@ -136,12 +144,28 @@ cd archer && python3 manage.py createsuperuser<br/>
 ![image](https://github.com/jly8866/archer/raw/master/screenshots/waitingforme.png)<br/>
 5. 用户登录页：<br/>
 ![image](https://github.com/jly8866/archer/raw/master/screenshots/login.png)<br/>
-6. 用户、集群、工单管理：<br/>
-![image](https://github.com/jly8866/archer/raw/master/screenshots/adminsqlusers.png)<br/>
-7. 工单统计图表：<br/>
-![image](https://github.com/jly8866/archer/raw/master/screenshots/charts.png)<br/><br/>
-8.pt-osc进度条，以及中止pt-osc进程按钮：<br/>
+6. 工单统计图表：<br/>
+![image](https://github.com/jly8866/archer/raw/master/screenshots/charts.png)<br/>
+7. pt-osc进度条，以及中止pt-osc进程按钮：<br/>
 ![image](https://raw.githubusercontent.com/johnliu2008/archer/master/screenshots/osc_progress.png)<br/>
+8. SQL在线查询、自动补全：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/query.png)<br/>
+9. 动态脱敏：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/datamasking.png)<br/>
+10. SQL在线查询日志：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/querylog.png)<br/>
+11. SQL在线查询权限申请：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/applyforprivileges.png)<br/>
+12. SQL慢查日志统计：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/slowquery.png)<br/>
+13. SQL慢查日志明细：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/slowquerylog.png)<br/>
+14. 阿里云RDS进程管理、表空间查询：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/process.png)<br/>
+15. SQLAdvisor：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/sqladvisor.png)<br/>
+15. 后台数据管理：<br/>
+![image](https://github.com/hhyo/archer/blob/master/src/screenshots/admin.png)<br/>
 
 ### 联系方式：
 QQ群：524233225

diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,3 @@
-aliyun-python-sdk-core==2.3.5
-aliyun-python-sdk-core-v3==2.5.3
-aliyun-python-sdk-rds==2.1.1
 certifi==2017.7.27.1
 chardet==3.0.4
 crypto==1.4.1
@@ -17,3 +14,6 @@ urllib3==1.22
 django-admin-bootstrapped==2.5.7
 gunicorn==19.7.1
 django-auth-ldap==1.3.0
+aliyun-python-sdk-core==2.3.5
+aliyun-python-sdk-core-v3==2.5.3
+aliyun-python-sdk-rds==2.1.1
diff --git a/sql/static/detail.html b/sql/static/detail.html
@@ -3,15 +3,6 @@
 {% block content %}
     <h4 style="display: inline;">单子名称：<span id="editWorkflowNname">{{ workflowDetail.workflow_name }}</span></h4>
     &nbsp;&nbsp;&nbsp;
-    {% if loginUserOb.is_superuser == 1 %}
-        <a type='button' id="btnViewSql" class='btn btn-default' onclick="loading(this)" href="/editsql/">查看提交信息</a>
-        {% if workflowDetail.status == '等待审核人审核' or workflowDetail.status == '自动审核不通过' or  workflowDetail.status == '审核通过' %}
-            <button class="btn btn-danger" data-toggle="modal" data-target="#executeComfirm">跳过inception执行
-            </button>
-        {% endif %}
-    {% elif loginUser in listAllReviewMen %}
-        <a type='button' id="btnViewSql" class='btn btn-default' onclick="loading(this)" href="/editsql/">查看提交信息</a>
-    {% endif %}
     <input type="hidden" id="workflowDetail_id" name="workflowid" value="{{ workflowDetail.id }}">
     <input type="hidden" id="editSqlContent" value="{{ workflowDetail.sql_content }}"/>
     <input type="hidden" id="editClustername" value="{{ workflowDetail.cluster_name }}"/>
@@ -80,25 +71,7 @@ <h4 style="display: inline;">单子名称：<span id="editWorkflowNname">{{ work
     <br>
     <table id="tb-detail" data-toggle="table" class="table table-condensed"></table>
 
-    {% if workflowDetail.is_manual == 1 and workflowDetail.execute_result %}
-        <br>
-        <table data-toggle="table" class="table table-striped table-hover">
-            <thead>
-            <tr>
-                <th>
-                    手工执行结果
-                </th>
-            </tr>
-            </thead>
-            <tbody>
-            <tr>
-                <td>
-                    {{ workflowDetail.execute_result }}
-                </td>
-            </tr>
-            </tbody>
-        </table>
-    {% endif %}
+
     {% if workflowDetail.audit_remark  and workflowDetail.status == '人工终止流程' %}
         <br>
         <table data-toggle="table" class="table table-striped table-hover">
@@ -124,11 +97,6 @@ <h4 style="display: inline;">单子名称：<span id="editWorkflowNname">{{ work
             <textarea id="remark" name="remark" class="form-control" data-name="审核备注"
                       placeholder="请填写驳回原因" rows=3></textarea>
             <br>
-            <form action="/passonly/" method="post" style="display:inline-block;">
-                {% csrf_token %}
-                <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
-                <input type="submit" id="btnPass" onclick="loading(this)" class="btn btn-success" value="审核通过"/>
-            </form>
 
             <form action="/execute/" method="post" style="display:inline-block;">
                 {% csrf_token %}
@@ -145,9 +113,6 @@ <h4 style="display: inline;">单子名称：<span id="editWorkflowNname">{{ work
             </form>
 
         {% elif workflowDetail.engineer == loginUser %}
-            <!--只允许发起人提交其他集群-->
-            <a type='button' id="btnSubmitOtherCluster" class='btn btn-warning' href="/submitothercluster/">上线其他集群</a>
-
             <form id="form-cancel" action="/cancel/" method="post" style="display:inline-block;">
                 {% csrf_token %}
                 <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
@@ -156,56 +121,20 @@ <h4 style="display: inline;">单子名称：<span id="editWorkflowNname">{{ work
             </form>
         {% endif %}
 
-    {% elif workflowDetail.status == '审核通过' %}
-        {% if loginUser in listAllReviewMen %}
-            <textarea id="remark" name="remark" class="form-control" data-name="审核备注"
-                      placeholder="请填写驳回原因" rows=3></textarea>
-            <br>
-
-            <form action="/executeonly/" method="post" style="display:inline-block;">
-                {% csrf_token %}
-                <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
-                <input type="submit" id="btnExecuteOnly" onclick="loading(this)" class="btn btn-danger"
-                       value="执行SQL"/>
-            </form>
-
-            <form id="form-cancel" action="/cancel/" method="post" style="display:inline-block;">
-                {% csrf_token %}
-                <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
-                <input type="hidden" id="audit_remark" name="audit_remark" value="">
-                <input type="button" id="btnCancel" class="btn btn-default" value="终止流程"/>
-            </form>
-        {% elif workflowDetail.engineer == loginUser %}
-            <!--只允许发起人提交其他集群-->
-            <a type='button' id="btnSubmitOtherCluster" class='btn btn-warning' href="/submitothercluster/">上线其他集群</a>
-
-            <form id="form-cancel" action="/cancel/" method="post" style="display:inline-block;">
-                {% csrf_token %}
-                <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
-                <input type="hidden" id="audit_remark" name="audit_remark" value="">
-                <input type="submit" onclick="loading(this)" class="btn btn-default" value="终止流程"/>
-            </form>
-        {% endif %}
 
     {% elif workflowDetail.status == '已正常结束' and workflowDetail.is_backup == '是' %}
         <form action="/rollback/" method="get" style="display:inline-block;">
             {% csrf_token %}
             <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
             <input type="submit" id="btnRollback" onclick="loading(this)" class="btn btn-default" value="查看回滚SQL"/>
         </form>
-        {% if workflowDetail.engineer == loginUser %}
-            <!--只允许发起人提交其他集群-->
-            <a type='button' id="btnSubmitOtherCluster" onclick="loading(this)" class='btn btn-warning'
-               href="/submitothercluster/">上线其他集群</a>
-        {% endif %}
+
 
     {% elif workflowDetail.status == '自动审核不通过' or workflowDetail.status == '人工终止流程' or workflowDetail.status == '执行有异常' %}
         {% if workflowDetail.engineer == loginUser %}
             <!--只允许发起人修改工单-->
             {% csrf_token %}
             <a type='button' id="btnEditSql" onclick="loading(this)" class='btn btn-warning' href="/editsql/">重新修改</a>
-            <a type='button' id="btnSubmitOtherCluster" onclick="loading(this)" class='btn btn-default'
-               href="/submitothercluster/">上线其他集群</a>
         {% endif %}
         {% if workflowDetail.status == '执行有异常' %}
             <form action="/rollback/" method="get" style="display:inline-block;">
@@ -216,32 +145,6 @@ <h4 style="display: inline;">单子名称：<span id="editWorkflowNname">{{ work
         {% endif %}
     {% endif %}
 
-    <!-- 手动执行确认 -->
-    <div class="modal fade" id="executeComfirm">
-        <div class="modal-dialog">
-            <div class="modal-content message_align">
-                <div class="modal-header ">
-                    <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span
-                            aria-hidden="true">×</span></button>
-                    <h4 class="modal-title text-danger">确定要手工执行吗？</h4>
-                </div>
-                <div class="modal-body">
-                    <p>手工执行将会跳过inception直接执行sql，无法生成备份，请谨慎操作！</p>
-                </div>
-                <div class="modal-footer">
-                    <button type="button" class="btn btn-info" data-dismiss="modal">取消</button>
-                    <form action="/execute_skipinc/" method="post" style="display:inline-block;">
-                        {% csrf_token %}
-                        <input type="hidden" name="workflowid" value="{{ workflowDetail.id }}">
-                        <input type="submit" id="btnExecuteSkipinc" onclick="loading(this)"
-                               class="btn btn-danger"
-                               value="确认执行"/>
-                    </form>
-                </div>
-            </div>
-        </div>
-    </div>
-
 {% endblock content %}
 
 {% block js %}

diff --git a/sql/static/submitSql.html b/sql/static/submitSql.html
@@ -114,8 +114,7 @@ <h4 class="modal-title text-danger">提交信息确认</h4>
                     <div class="modal-body">
                         <p>提交的SQL经检测仍存在<font color="red" size="5"><b id="CheckWarningCount"></b></font>个警告信息和<font
                                 color="red" size="5"><b id="CheckErrorCount"></b></font>个错误信息<br>
-                            存在警告信息可能会被审核人打回，请按照平台规范仔细检查<br>
-                            存在错误信息将被直接审核不通过，如果确认语句正确请在提交后联系审核人操作执行</p>
+                            请按照平台规范仔细检查！<br>
                     </div>
                     <div class="modal-footer">
                         <button type="button" class="btn btn-info" data-dismiss="modal">取消</button>

diff --git a/sql/static/user/js/autoreview.js b/sql/static/user/js/autoreview.js
@@ -114,8 +114,8 @@ function autoreview() {
                         pagination: true,                   //是否显示分页（*）
                         sidePagination: "client",           //分页方式：client客户端分页，server服务端分页（*）
                         pageNumber: 1,                      //初始化加载第一页，默认第一页,并记录
-                        pageSize: 30,                     //每页的记录行数（*）
-                        pageList: [30, 50, 100],       //可供选择的每页的行数（*）
+                        pageSize: 500,                     //每页的记录行数（*）
+                        pageList: [500, 1000, 10000],       //可供选择的每页的行数（*）
                         search: false,                      //是否显示表格搜索
                         strictSearch: false,                //是否全匹配搜索
                         showColumns: true,                  //是否显示所有的列（选择显示的列）

diff --git a/sql/urls.py b/sql/urls.py
@@ -9,16 +9,12 @@
     url(r'^login/$', views.login, name='login'),
     url(r'^logout/$', views.logout, name='logout'),
     url(r'^submitsql/$', views.submitSql, name='submitSql'),
-    url(r'^editsql/$', views.submitSql, name='editsql'),
-    url(r'^submitothercluster/$', views.submitSql, name='submitothercluster'),
+    url(r'editsql/$', views.submitSql, name='editsql'),
     url(r'^allworkflow/$', views.allworkflow, name='allworkflow'),
 
     url(r'^autoreview/$', views.autoreview, name='autoreview'),
     url(r'^detail/(?P<workflowId>[0-9]+)/$', views.detail, name='detail'),
-    url(r'^passonly/$', views.passonly, name='passonly'),
-    url(r'^executeonly/$', views.executeonly, name='executeonly'),
     url(r'^execute/$', views.execute, name='execute'),
-    url(r'^execute_skipinc/$', views.execute_skipinc, name='execute_skipinc'),
     url(r'^cancel/$', views.cancel, name='cancel'),
     url(r'^rollback/$', views.rollback, name='rollback'),
     url(r'^ldapsync/$', views.ldapsync, name='ldapsync'),