Skip to content

Commit

Permalink
首次提交
Browse files Browse the repository at this point in the history
  • Loading branch information
@shawnloong
shawnloong committed May 11, 2020
0 parents commit 309c1bf
Show file tree
Hide file tree
Showing 8 changed files with 248 additions and 0 deletions.
Binary file added __pycache__/db_conn.cpython-37.pyc
Binary file not shown.
149 changes: 149 additions & 0 deletions archive_tab_exec.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''=================================================
@Project -> File ：mysql_archive_db -> archive_tab_exec
@IDE ：PyCharm
@Author ：Netdata
@Date ：2020/5/10 22:02
@Desc ：
=================================================='''
import sys
import os
import time
import db_conn
import subprocess
from subprocess import Popen
import datetime

db = db_conn.conn
cursor = db.cursor()

try:
sql = "SELECT id,db_nick_name,source_db_server,source_db_port,source_user,source_passwd,source_schema, \
source_tab,source_charset,dest_db_server,dest_db_port,dest_user,dest_passwd,dest_schema,dest_tab,dest_charset, \
archive_condition,is_archive FROM ops.archive_tab_info where is_archive=1"

cursor.execute(sql)
results = cursor.fetchall()
for row in results:
v_id = row[0]
v_nick_name = row[1]
v_source_db_server = row[2]
v_source_db_port = row[3]
v_source_user = row[4]
v_source_passwd = row[5]
v_source_schema = row[6]
v_source_tab = row[7]
v_source_charset = row[8]
v_dest_db_server = row[9]
v_dest_db_port = row[10]
v_dest_user = row[11]
v_dest_passwd = row[12]
v_dest_schema = row[13]
v_dest_tab = row[14]
v_dest_charset = row[15]
v_archive_condition = row[16]
v_is_archive = row[17]

archive_cmd_utf8mb4 = "pt-archiver " \
"--source h='%s',P='%s',u='%s',p='%s',D='%s',t='%s',A=utf8mb4 " \
"--dest h='%s',P='%s',u='%s',p='%s',D='%s',t='%s',A=utf8mb4 " \
"--where '%s' --progress 5000 --limit 1000 --txn-size 1000 " \
"--statistics --purge " % \
(v_source_db_server, v_source_db_port, v_source_user, v_source_passwd, v_source_schema,v_source_tab, \
v_dest_db_server, v_source_db_port, v_dest_user, v_dest_passwd, v_dest_schema, v_dest_tab, \
v_archive_condition)

archive_cmd_utf8 = "pt-archiver " \
"--source h='%s',P='%s',u='%s',p='%s',D='%s',t='%s' " \
"--dest h='%s',P='%s',u='%s',p='%s',D='%s',t='%s' " \
"--charset=UTF8 --where '%s' --progress 5000 --limit 1000 --txn-size 1000 --bulk-insert --bulk-delete " \
"--statistics --purge " % \
(v_source_db_server, v_source_db_port, v_source_user, v_source_passwd, v_source_schema,v_source_tab, \
v_dest_db_server, v_source_db_port, v_dest_user, v_dest_passwd, v_dest_schema, v_dest_tab, \
v_archive_condition)
print(archive_cmd_utf8mb4)
log_file_name = "./logs/db_archive_%s_%s.log" % (v_source_db_server, v_source_tab)
myoutput = open(log_file_name,'w+')

#判断字符集
if v_source_charset == 'utf8mb4':
archive_starttime = datetime.datetime.now()
p = subprocess.Popen(archive_cmd_utf8mb4, shell=True, stdout=myoutput, stderr=myoutput, universal_newlines=True)
output, errors = p.communicate()
with open(log_file_name,"r") as f:
print(f.read())
archive_endtime = datetime.datetime.now()
#计算时间
print(archive_starttime,archive_endtime)
v_cost_time = (archive_starttime - archive_endtime).seconds

inserted_qty = 0
deleted_qty = 0
with open(log_file_name,"r") as f:
for line in f:
if 'INSERT' in line:
i = line.index(" ")
inserted_qty = line[i+1:]
elif 'DELETE' in line:
i = line.index(" ")
deleted_qty = line[i+1:]

if inserted_qty == deleted_qty:
archive_status = 1
else:
archive_status = 0

#记录log
sql_insert = "insert into archive_tab_log(dbid, db_nick_name, archive_starttime, archive_endtime, " \
"archive_cmd, archive_status, archive_qty, cost_time ) " \
"values('%s','%s','%s','%s','%s','%s','%s','%s')" % \
(v_id, v_nick_name, archive_starttime, archive_endtime, \
db.escape_string(archive_cmd_utf8mb4), archive_status, inserted_qty, v_cost_time)
#print(sql_insert)
cursor.execute(sql_insert)
# exec commit
db.commit()

elif v_source_charset == 'utf8':
archive_starttime = datetime.datetime.now()
p = subprocess.Popen(archive_cmd_utf8, shell=True, stdout=myoutput, stderr=myoutput, universal_newlines=True)
output, errors = p.communicate()
with open(log_file_name,"r") as f:
print(f.read())
archive_endtime = datetime.datetime.now()
v_cost_time = (archive_starttime - archive_endtime).seconds

inserted_qty = 0
deleted_qty = 0
with open(log_file_name,"r") as f:
for line in f:
if 'INSERT' in line:
i = line.index(" ")
inserted_qty = line[i+1:]
elif 'DELETE' in line:
i = line.index(" ")
deleted_qty = line[i+1:]

if inserted_qty == deleted_qty:
archive_status = 1
else:
archive_status = 0

#记录log
sql_insert = "insert into archive_tab_log(dbid, db_nick_name, archive_starttime, archive_endtime, " \
"archive_cmd, archive_status, archive_qty, cost_time ) " \
"values('%s','%s','%s','%s','%s','%s','%s','%s')" % \
(v_id, v_nick_name, archive_starttime, archive_endtime, \
db.escape_string(archive_cmd_utf8), archive_status, inserted_qty, v_cost_time)
cursor.execute(sql_insert)
# exec commit
db.commit()

else:
print("charset is error!!!!")

except Exception as e:
raise e
finally:
db.close()
21 changes: 21 additions & 0 deletions db_conn.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''=================================================
@Project -> File ：mysql_archive_db -> db_conn.py
@IDE ：PyCharm
@Author ：Netdata
@Date ：2020/5/10 19:18
@Desc ：
=================================================='''

import pymysql
import sys
import datetime

conn = pymysql.connect(
host="192.168.137.3",
port=3306,
user="netdata",
password="netdata",
database="ops",
charset="utf8")
Empty file added logs/archive_tab.log
Empty file.
20 changes: 20 additions & 0 deletions logs/db_archive_192.168.137.3_user_info.log
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
TIME ELAPSED COUNT
2020-05-11T21:58:12 0 0
2020-05-11T21:58:23 10 5000
2020-05-11T21:58:34 21 10000
2020-05-11T21:58:45 32 15000
2020-05-11T21:58:56 43 20000
2020-05-11T21:59:07 54 25000
2020-05-11T21:59:15 62 28628
Started at 2020-05-11T21:58:12, ended at 2020-05-11T21:59:15
Source: A=utf8mb4,D=netdata,P=3306,h=192.168.137.3,p=...,t=user_info,u=netdata
Dest: A=utf8mb4,D=netdata,P=3306,h=192.168.137.3,p=...,t=his_user_info,u=netdata
SELECT 28628
INSERT 28628
DELETE 28628
Action Count Time Pct
deleting 28628 25.9422 41.39
inserting 28628 21.9006 34.94
select 30 9.7683 15.59
commit 58 0.3684 0.59
other 0 4.6929 7.49
19 changes: 19 additions & 0 deletions logs/db_archive_192.168.137.3_utf8_userinfo.log
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
TIME ELAPSED COUNT
2020-05-11T21:59:16 0 0
2020-05-11T21:59:19 2 5000
2020-05-11T21:59:20 4 10000
2020-05-11T21:59:22 5 15000
2020-05-11T21:59:22 6 16496
Started at 2020-05-11T21:59:16, ended at 2020-05-11T21:59:22
Source: A=UTF8,D=netdata,P=3306,h=192.168.137.3,p=...,t=utf8_userinfo,u=netdata
Dest: A=UTF8,D=netdata,P=3306,h=192.168.137.3,p=...,t=his_utf8_userinfo,u=netdata
SELECT 16496
INSERT 16496
DELETE 16496
Action Count Time Pct
bulk_deleting 17 1.7895 28.28
select 18 1.2431 19.64
bulk_inserting 17 1.0611 16.77
commit 34 0.2587 4.09
print_bulkfile 16496 -0.0015 -0.02
other 0 1.9771 31.24
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
PyMySQL==0.9.3
38 changes: 38 additions & 0 deletions sql/create_table.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
CREATE TABLE archive_tab_info (
id bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
db_nick_name varchar(256) DEFAULT NULL COMMENT 'DB名称',
source_db_server varchar(128) DEFAULT NULL COMMENT '源DB服务器',
source_db_port int(11) DEFAULT NULL COMMENT '源DB端口',
source_user varchar(64) DEFAULT NULL COMMENT '源DB用户名',
source_passwd varchar(128) DEFAULT NULL COMMENT '源DB密码',
source_schema varchar(128) DEFAULT NULL COMMENT '源DB名称',
source_tab varchar(128) DEFAULT NULL COMMENT '源归档表名',
source_charset varchar(16) DEFAULT NULL COMMENT '源端字符集',
dest_db_server varchar(128) DEFAULT NULL COMMENT '目标端服务器',
dest_db_port int(11) DEFAULT NULL COMMENT '目标端服务器端口',
dest_user varchar(64) DEFAULT NULL COMMENT '目标端DB用户名',
dest_passwd varchar(128) DEFAULT NULL COMMENT '目标端服务器密码',
dest_schema varchar(128) DEFAULT NULL COMMENT '目标DB名称',
dest_tab varchar(128) DEFAULT NULL COMMENT '目标端服务器表名',
dest_charset varchar(16) DEFAULT NULL COMMENT '目标端字符集',
archive_condition varchar(1024) DEFAULT NULL COMMENT '归档条件',
is_archive int(11) DEFAULT '0' COMMENT '是否归档,1归档，0不归档',
create_time datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
modify_time datetime DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
PRIMARY KEY (id)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;

CREATE TABLE archive_tab_log (
id bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
dbid bigint(20) DEFAULT NULL COMMENT 'dbid关联archive_tab_info id',
db_nick_name varchar(128) DEFAULT NULL COMMENT '归档DB名称',
archive_starttime datetime DEFAULT NULL COMMENT '归档开始时间',
archive_endtime datetime DEFAULT NULL COMMENT '归档结束时间',
archive_cmd varchar(2048) DEFAULT NULL COMMENT '归档命令内容',
archive_status int(11) DEFAULT NULL COMMENT '归档状态,0失败1成功',
archive_qty bigint(20) DEFAULT NULL COMMENT '归档行数',
cost_time bigint(20) DEFAULT NULL COMMENT '花费时间秒',
create_time datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
modify_time datetime DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
PRIMARY KEY (id)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;

0 comments on commit 309c1bf

Please sign in to comment.