博客
关于我
强烈建议你试试无所不能的chatGPT,快点击我
MySQL python 数据迁移脚本
阅读量:6703 次
发布时间:2019-06-25

本文共 5022 字,大约阅读时间需要 16 分钟。

用python写了个数据迁移脚本,主要是利用从库将大的静态表导出表空间,载导入到目标实例中。

#!/usr/bin/env python3#-*- coding:utf8 -*-#author:zhanbin.liu#!!!!!DB必须同版本#python3环境   pip3 install pymysql paramikoimport os#from pathlib import Pathimport sysimport pymysqlimport paramiko#每次只能迁移一个DB下的表,到指定DB#GRANT SELECT, CREATE, RELOAD, ALTER, LOCK TABLES ON *.* TO 'data_migration'@'192.168.%' IDENTIFIED BY  'data_migration@123';tables='sqlauto_cluster,sqlauto_user'       #以,分割的字符串,如a,b,ctableList = tables.split(',')sourceIp = '192.168.1.101'sourceDataBase = '/data/mysql/3306/data'sourceDbName = 'inception_web'sourceDataDir = os.path.join(sourceDataBase,sourceDbName)desIp = '192.168.1.102'desDataBase = '/data/mysql/3306/data'desDbName = 'inception_web'desDataDir = os.path.join(desDataBase,desDbName)# for table in tableList:#     desFile = Path("%s/%s.ibd" %(desDataDir,table))#     print(desFile)#     if desFile.is_file():#         print("ok")#     else:#         print("no")comUser = 'data_migration'comPwd = 'data_migration@123'comPort = 3306client = paramiko.SSHClient()client.set_missing_host_key_policy(paramiko.AutoAddPolicy())def table_judge():    print("table_judge")    sourceTableExist = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')    desTableExist = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')    sourceTables = []    desTables = []    cursor_source = sourceTableExist.cursor()    cursor_des = desTableExist.cursor()    for table in tableList:        #print(table)        cursor_source.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (sourceDbName,table))        sourceTable_tmp = cursor_source.fetchall()        cursor_des.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (desDbName,table))        desTable_tmp = cursor_des.fetchall()        #print(desTable_tmp)        if sourceTable_tmp is ():            sourceTables.append(table)        if desTable_tmp is not ():            desTables.append(desTable_tmp[0][0])    sourceTableExist.close()    desTableExist.close()    s=d=0    if sourceTables != []:        print('迁移源不存在将要迁移的表:',sourceIp,sourceDbName, sourceTables,' 请检查')        s=1    if desTables != []:        print('目标库存在将要迁移的表:',desIp,desDbName,desTables,' 请移除')        d=1    if s == 1 or d == 1:        sys.exit()def data_sync():    print('data_sync')    db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')    db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')    cursor_db_source = db_source.cursor()    cursor_db_des = db_des.cursor()    for table in tableList:        print("正在同步表:",table)        cursor_db_source.execute("show create table %s;" % (table))        createTableSQL = cursor_db_source.fetchall()[0][1]        print(createTableSQL)        try:            cursor_db_des.execute(createTableSQL)        except Exception as error:            print(error)        cursor_db_source.execute("flush table %s with read lock;" % (table))        cursor_db_des.execute("alter table %s discard tablespace;" % (table))        client.connect(sourceIp, 22, 'root')        stdin1, stdout1, stderr1 = client.exec_command("scp %s  %s:%s " % (sourceDataDir+"/"+table+".ibd", desIp, desDataDir))        stdin2, stdout2, stderr2 = client.exec_command("scp %s  %s:%s " % (sourceDataDir+"/"+table+".cfg", desIp, desDataDir))        a_e_1 = stderr1.readlines()        a_e_2 = stderr2.readlines()        if a_e_1 != [] or a_e_2 != []:            print(a_e_1,a_e_2)            sys.exit()        client.close()        client.connect(desIp, 22, 'root')        stdin3, stdout3, stderr3 = client.exec_command("chown -R mysql.mysql %s*" % (desDataDir+"/"+table))        a_e_3 = stderr3.readlines()        if a_e_3 != []:            print(a_e_1, a_e_2)            sys.exit()        client.close()        #cursor_db_source.execute("select sleep(10);")        cursor_db_source.execute("unlock tables;")        cursor_db_des.execute("alter table %s import tablespace;" % (table))        print("同步完成")    cursor_db_source.close()    cursor_db_des.close()def data_checksum():    print('data_checksum')    db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')    db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')    cursor_db_source = db_source.cursor()    cursor_db_des = db_des.cursor()    for table in tableList:        print("正在校验表:", table)        cursor_db_source.execute("checksum table %s;" % (table))        ck_s = cursor_db_source.fetchall()[0][1]        cursor_db_des.execute("checksum table %s;" % (table))        ck_d = cursor_db_des.fetchall()[0][1]        if ck_s != ck_d:            print("表不一致:",table)        else:            print("表一致:",table)    cursor_db_source.close()    cursor_db_des.close()if __name__ == "__main__":    table_judge()    data_sync()    data_checksum()    print('haha')

转载于:https://blog.51cto.com/liuzhanbin/2337168

你可能感兴趣的文章
C++语言中volatile关键字的作用
查看>>
01_04 JSP基础语法之forward
查看>>
VMware嵌套虚拟化
查看>>
CPP 调用简单的汇编函数
查看>>
Golang Slice interface and array concatenation
查看>>
java nio
查看>>
nginx反向代理配置及优化
查看>>
linux下使用free命令查看实际内存占用
查看>>
整合 Tachyon 运行 Spark(译)
查看>>
外部链接增加nofllow
查看>>
mysql update safe model 一些问题
查看>>
Java项目开发环境构建工具 Gradle 使用笔记(简单、基本)
查看>>
Scala的模式匹配和条件类
查看>>
Scala函数与对象
查看>>
前端Jquery小结
查看>>
浅谈Java的输入输出流
查看>>
Eclipse安装SVN插件
查看>>
gradle3.1初步了解
查看>>
网易招聘笔试题+答案解析
查看>>
atoi和itoa函数的实现
查看>>