Merge pull request jly8866#1 from jly8866/master

更新master代码

README.md

@@ -91,6 +91,18 @@ cd archer && python3 manage.py createsuperuser<br/>
 这一步是为了告诉archer你要用inception去哪些mysql主库里执行SQL，所用到的用户名密码、端口等。<br/>
 11. 正式访问：<br/>
 以上步骤完毕，就可以使用步骤9创建的用户登录archer系统啦, 首页地址 http://X.X.X.X:port/<br/>
+<br/>
+如果觉得以上安装步骤还是看不懂，可以看这一篇安装步骤，感谢网友@一条大河 的贡献：https://riverdba.github.io/2017/04/15/archer-install/ <br/>
+
+### 集成ldap
+1. settings中ENABLE_LDAP改为True,可以启用ldap账号登陆<br/>
+2. 如果使用了ldaps，并且是自签名证书，需要打开settings中AUTH_LDAP_GLOBAL_OPTIONS的注释<br/>
+3. centos需要执行yum install openldap-devel<br/>
+4. settings中以AUTH_LDAP开头的配置，需要根据自己的ldap对应修改<br/>
+
+### admin后台加固，防暴力破解
+1.patch目录下，名称为：django_1.8.17_admin_secure_archer.patch
+2.使用命令：patch  python/site-packages/django/contrib/auth/views.py django_1.8.17_admin_secure_archer.patch
 
 ### 已经制作好的docker镜像：
 * 如果不想自己安装上述，可以直接使用做好的docker镜像，安装步骤：

archer/settings.py

@@ -128,6 +128,91 @@
 INCEPTION_REMOTE_BACKUP_USER='inception'
 INCEPTION_REMOTE_BACKUP_PASSWORD='inception'
 
+# 账户登录失败锁定时间(秒)
+LOCK_TIME_THRESHOLD = 300
+# 账户登录失败 几次 锁账户
+LOCK_CNT_THRESHOLD = 5
+
+# LDAP
+ENABLE_LDAP = False
+if ENABLE_LDAP:
+    import ldap
+    # from django_auth_ldap.config import LDAPSearch, GroupOfNamesType
+    from django_auth_ldap.config import LDAPSearch, GroupOfUniqueNamesType
+    # if use self signed certificate, Remove AUTH_LDAP_GLOBAL_OPTIONS annotations
+    #AUTH_LDAP_GLOBAL_OPTIONS={
+    #    ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER
+    #}
+    AUTH_LDAP_BIND_DN = "cn=ro,dc=xxx,dc=cn"
+    AUTH_LDAP_BIND_PASSWORD = "xxxxxx"
+    AUTH_LDAP_SERVER_URI = "ldap://auth.xxx.com"
+    AUTH_LDAP_BASEDN = "ou=users,dc=xxx,dc=cn"
+    AUTH_LDAP_USER_DN_TEMPLATE = "cn=%(user)s,ou=users,dc=xxx,dc=cn"
+    AUTH_LDAP_GROUP_SEARCH = LDAPSearch("ou=groups,dc=xxx,dc=cn",
+        ldap.SCOPE_SUBTREE, "(objectClass=groupOfUniqueNames)"
+    )
+    AUTH_LDAP_GROUP_TYPE = GroupOfUniqueNamesType()
+    AUTH_LDAP_USER_ATTRLIST = ["cn", "sn", "mail"]
+    AUTH_LDAP_USER_ATTR_MAP = {
+        "username": "cn",
+        "display": "sn",
+        "email": "mail"
+    }
+
+    # AUTH_LDAP_MIRROR_GROUPS = True  # 直接把ldap的组复制到django一份，和AUTH_LDAP_FIND_GROUP_PERMS互斥.用户每次登录会根据ldap来更新数据库的组关系
+    # AUTH_LDAP_FIND_GROUP_PERMS = True  # django从ldap的组权限中获取权限,这种方式，django自身不创建组，每次请求都调用ldap
+    # AUTH_LDAP_CACHE_GROUPS = True  # 如打开FIND_GROUP_PERMS后，此配置生效，对组关系进行缓存，不用每次请求都调用ldap
+    # AUTH_LDAP_GROUP_CACHE_TIMEOUT = 600  # 缓存时间
+
+#开启以下配置注释，可以帮助调试ldap集成
+LDAP_LOGS = '/tmp/ldap.log'
+DEFAULT_LOGS = '/tmp/default.log'
+stamdard_format = '[%(asctime)s][%(threadName)s:%(thread)d]' + \
+                  '[task_id:%(name)s][%(filename)s:%(lineno)d] ' + \
+                  '[%(levelname)s]- %(message)s'
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'standard': {  # 详细
+            'format': stamdard_format
+        },
+    },
+    'handlers': {
+        'default': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': DEFAULT_LOGS,
+            'maxBytes': 1024 * 1024 * 100,  # 5 MB
+            'backupCount': 5,
+            'formatter': 'standard',
+        },
+        'ldap': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': LDAP_LOGS,
+            'maxBytes': 1024 * 1024 * 100,  # 5 MB
+            'backupCount': 5,
+            'formatter': 'standard',
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+        }
+    },
+    'loggers': {
+        'default': {  # default日志，存放于log中
+            'handlers': ['default'],
+            'level': 'DEBUG',
+        },
+        'django_auth_ldap': {  # django_auth_ldap模块相关日志打印到console
+            'handlers': ['ldap'],
+            'level': 'DEBUG',
+            'propagate': True,  # 选择关闭继承，不然这个logger继承自默认，日志就会被记录2次了(''一次，自己一次)
+        }
+    }
+}
+
 #是否开启邮件提醒功能：发起SQL上线后会发送邮件提醒审核人审核，执行完毕会发送给DBA. on是开，off是关，配置为其他值均会被archer认为不开启邮件功能
 MAIL_ON_OFF='on'
 
@@ -136,6 +221,7 @@
 MAIL_REVIEW_FROM_ADDR='[email protected]'                                               #发件人，也是登录SMTP server需要提供的用户名
 MAIL_REVIEW_FROM_PASSWORD=''                                                         #发件人邮箱密码，如果为空则不需要login SMTP server
 MAIL_REVIEW_DBA_ADDR=['[email protected]', '[email protected]']        #DBA地址，执行完毕会发邮件给DBA，以list形式保存
+MAIL_REVIEW_SECURE_ADDR=['[email protected]', '[email protected]']     #登录失败，等安全相关发送地址
 #是否过滤【DROP DATABASE】|【DROP TABLE】|【TRUNCATE PARTITION】|【TRUNCATE TABLE】等高危DDL操作：
 #on是开，会首先用正则表达式匹配sqlContent，如果匹配到高危DDL操作，则判断为“自动审核不通过”；off是关，直接将所有的SQL语句提交给inception，对于上述高危DDL操作，只备份元数据
 CRITICAL_DDL_ON_OFF='off'

patch/django_1.8.17_admin_secure_archer.patch

@@ -0,0 +1,59 @@
+--- views.py	2018-01-23 11:53:00.179201491 +0800
++++ python/site-packages/django/contrib/auth/views.py	2018-01-23 11:58:10.668286140 +0800
+@@ -24,7 +24,14 @@
+ from django.views.decorators.cache import never_cache
+ from django.views.decorators.csrf import csrf_protect
+ from django.views.decorators.debug import sensitive_post_parameters
+-
++# 账户锁定
++from django.conf import settings
++from sql.sendmail import MailSender
++import datetime
++import logging
++logger = logging.getLogger('default')
++login_failure_counter = {}
++# 账户锁定end
+
+ @sensitive_post_parameters()
+ @csrf_protect
+@@ -41,8 +48,22 @@
+
+     if request.method == "POST":
+         form = authentication_form(request, data=request.POST)
+-        if form.is_valid():
+-
++        
++        # 增加账户锁定
++        failed_cnt = settings.LOCK_CNT_THRESHOLD
++        locking_time = settings.LOCK_TIME_THRESHOLD
++        username = request.POST['username']
++        mailSender = MailSender()
++        now_time = datetime.datetime.now()
++        mail_title = 'login inception admin'
++        login_failed_message = ''
++        
++        if username in login_failure_counter and login_failure_counter[username]['cnt'] >= failed_cnt and (now_time - login_failure_counter[username]["last_failure_time"]).seconds <= locking_time:
++            login_failed_message = 'user:{},login /admin failed, account locking...'.format(username)
++            logger.warning(login_failed_message)
++            mailSender.sendEmail(mail_title, login_failed_message, getattr(settings, 'MAIL_REVIEW_SECURE_ADDR'))
++        elif form.is_valid():
++            logger.info('user:{},login /admin success'.format(username))
+             # Ensure the user-originating redirection url is safe.
+             if not is_safe_url(url=redirect_to, host=request.get_host()):
+                 redirect_to = resolve_url(settings.LOGIN_REDIRECT_URL)
+@@ -51,6 +72,15 @@
+             auth_login(request, form.get_user())
+
+             return HttpResponseRedirect(redirect_to)
++        else:
++            if username in login_failure_counter and (now_time - login_failure_counter[username]["last_failure_time"]).seconds <= locking_time:
++                login_failure_counter[username]["cnt"] += 1
++            else:
++                login_failure_counter[username] = {"cnt":1, "last_failure_time": datetime.datetime.now()}
++            login_failed_message = 'user:{},login /admin failed, fail count:{}'.format(username, login_failure_counter[username]["cnt"])
++            logger.warning(login_failed_message)
++            mailSender.sendEmail(mail_title, login_failed_message, getattr(settings, 'MAIL_REVIEW_SECURE_ADDR'))
++        #账户锁定end        
+     else:
+         form = authentication_form(request)
+

requirements.txt

@@ -1,2 +1,3 @@
 Django==1.8.17
 python==3.4.1
+django-auth-ldap==1.3.0

sql/inception.py

@@ -32,7 +32,7 @@ def criticalDDL(self, sqlContent):
         resultList = []
         criticalSqlFound = 0
         for row in sqlContent.rstrip(';').split(';'):
-            if re.match(r"([\s\S]*)drop(\s+)database(\s+.*)|([\s\S]*)drop(\s+)table(\s+.*)|([\s\S]*)truncate(\s+)partition(\s+.*)|([\s\S]*)truncate(\s+)table(\s+.*)", row.lower()):
+            if re.match(r"([\s\S]*)drop(\s+)database(\s+.*)|([\s\S]*)drop(\s+)table(\s+.*)|([\s\S]*)truncate(\s+.*)|([\s\S]*)truncate(\s+)partition(\s+.*)|([\s\S]*)truncate(\s+)table(\s+.*)", row.lower()):
                 result = ('', '', 2, '驳回高危SQL', '不能包含【DROP DATABASE】|【DROP TABLE】|【TRUNCATE PARTITION】|【TRUNCATE TABLE】关键字！', row, '', '', '', '')
                 criticalSqlFound = 1
             else:
@@ -236,4 +236,4 @@ def stopOscProgress(self, sqlSHA1):
             optResult = {"status":0, "msg":"已成功停止OSC进程，请注意清理触发器和临时表", "data":""}
         else:
             optResult = {"status":1, "msg":"ERROR 2624 (HY000):未找到OSC执行进程，可能已经执行完成", "data":""}
-        return optResult
+        return optResult

sql/models.py

@@ -11,13 +11,15 @@
 class users(AbstractUser):
     display = models.CharField('显示的中文名', max_length=50)
     role = models.CharField('角色', max_length=20, choices=(('工程师','工程师'),('审核人','审核人')), default='工程师')
+    is_ldapuser = models.BooleanField('ldap用戶', default=False)
 
     def __str__(self):
         return self.username
 
     class Meta:
         verbose_name = u'用户配置'
         verbose_name_plural = u'用户配置'
+users._meta.get_field('is_active').default = False # ldap default can't login, need admin to control
 
 #各个线上主库地址。
 class master_config(models.Model):

sql/processor.py

@@ -1,5 +1,7 @@
 # -*- coding: UTF-8 -*- 
 from .models import users
+from django.conf import settings
+
 leftMenuBtnsCommon = (
                    {'key':'allworkflow',        'name':'查看历史工单',     'url':'/allworkflow/',              'class':'glyphicon glyphicon-home'},
                    {'key':'submitsql',          'name':'发起SQL上线',       'url':'/submitsql/',               'class':'glyphicon glyphicon-asterisk'},
@@ -14,6 +16,14 @@
                    {'key':'charts',     'name':'统计图表展示',       'url':'/charts/',        'class':'glyphicon glyphicon-file'},
 )
 
+if settings.ENABLE_LDAP:
+    leftMenuBtnsSuper = (
+        {'key': 'masterconfig', 'name': '主库地址配置', 'url': '/admin/sql/master_config/', 'class': 'glyphicon glyphicon-user'},
+        {'key': 'userconfig', 'name': '用户权限配置', 'url': '/admin/sql/users/', 'class': 'glyphicon glyphicon-th-large'},
+        {'key': 'ldapsync', 'name': '同步LDAP用户', 'url': '/ldapsync/', 'class': 'glyphicon glyphicon-th-large'},
+        {'key': 'workflowconfig', 'name': '所有工单管理', 'url': '/admin/sql/workflow/','class': 'glyphicon glyphicon-list-alt'},
+    )
+
 def global_info(request):
     """存放用户，会话信息等."""
     loginUser = request.session.get('login_username', None)

sql/sendmail.py

@@ -21,6 +21,7 @@ def __init__(self):
             self.MAIL_REVIEW_FROM_ADDR = getattr(settings, 'MAIL_REVIEW_FROM_ADDR')
             self.MAIL_REVIEW_FROM_PASSWORD = getattr(settings, 'MAIL_REVIEW_FROM_PASSWORD')
             self.MAIL_REVIEW_DBA_ADDR = getattr(settings, 'MAIL_REVIEW_DBA_ADDR')
+            self.MAIL_REVIEW_SECURE_ADDR = getattr(settings, 'MAIL_REVIEW_SECURE_ADDR')
 
         except AttributeError as a:
             print("Error: %s" % a)

sql/static/base.html

@@ -119,6 +119,7 @@ <h4 class="modal-title" id="myModalLabel">
 	<script src="{% static 'user/js/submitsql.js' %}"></script>
 	<script src="{% static 'highcharts/highcharts.js' %}"></script>
 	<script src="{% static 'user/js/charts.js' %}"></script>
+	<script src="{% static 'user/js/ldapsync.js' %}"></script>
 </body>
 
 </html>

sql/static/ldapsync.html

@@ -0,0 +1,5 @@
+{% extends "base.html" %}
+
+{% block content %}
+<button type="button" id="btnSync" class="btn btn-primary">同步LDAP用户</button>
+{% endblock content %}

sql/static/user/js/ldapsync.js

@@ -0,0 +1,23 @@
+$('#btnSync').click(function(){
+	syncUser();
+});
+
+function syncUser() {
+    $.ajax({
+        type: "post",
+        url: "/syncldapuser/",
+        dataType: "json",
+        data: {},
+        complete: function () {
+        },
+        success: function (data) {
+			$('#wrongpwd-modal-body').html(data.msg);
+			$('#wrongpwd-modal').modal({
+				keyboard: true
+			});
+		},
+		error: function (XMLHttpRequest, textStatus, errorThrown) {
+			alert(errorThrown);
+        }
+    });
+};

sql/urls.py

@@ -17,10 +17,12 @@
     url(r'^execute/$', views.execute, name='execute'),
     url(r'^cancel/$', views.cancel, name='cancel'),
     url(r'^rollback/$', views.rollback, name='rollback'),
+    url(r'^ldapsync/$', views.ldapsync, name='ldapsync'),
     url(r'^dbaprinciples/$', views.dbaprinciples, name='dbaprinciples'),
     url(r'^charts/$', views.charts, name='charts'),
 
     url(r'^authenticate/$', views_ajax.authenticateEntry, name='authenticate'),
+    url(r'^syncldapuser/$', views_ajax.syncldapuser, name='syncldapuser'),
     url(r'^simplecheck/$', views_ajax.simplecheck, name='simplecheck'),
     url(r'^getMonthCharts/$', views_ajax.getMonthCharts, name='getMonthCharts'),
     url(r'^getPersonCharts/$', views_ajax.getPersonCharts, name='getPersonCharts'),

sql/views.py

@@ -213,7 +213,7 @@ def autoreview(request):
                 for reviewMan in listAllReviewMen:
                     if reviewMan == "":
                         continue
-                    strContent = "发起人：" + engineer + "\n审核人：" + reviewMan  + "\n工单地址：" + url + "\n工单名称： " + workflowName + "\n具体SQL：" + sqlContent
+                    strContent = "发起人：" + engineer + "\n审核人：" + str(listAllReviewMen)  + "\n工单地址：" + url + "\n工单名称： " + workflowName + "\n具体SQL：" + sqlContent
                     objReviewMan = users.objects.get(username=reviewMan)
                     mailSender.sendEmail(strTitle, strContent, [objReviewMan.email])
             else:
@@ -391,6 +391,11 @@ def rollback(request):
     context = {'listBackupSql':listBackupSql, 'rollbackWorkflowName':rollbackWorkflowName, 'cluster_name':cluster_name, 'review_man':review_man, 'sub_review_man':sub_review_man}
     return render(request, 'rollback.html', context)
 
+#ldap用户同步
+def ldapsync(request):
+    context = {'currentMenu':'ldapsync'}
+    return render(request, 'ldapsync.html', context)
+
 #SQL审核必读
 def dbaprinciples(request):
     context = {'currentMenu':'dbaprinciples'}