创建 config.ini `#需要合并的服务器 最大2 db_app_count = 1
需要合并资源1
db_host1 = 127.0.0.1 db_user1 = root db_pass1 = root
需要合并资源2
db_host2 = 127.0.0.1 db_user2 = root db_pass2 = root
目的地
db_host = 127.0.0.1 db_user = root db_pass = root #需要合并的表 db_list = []
#新服serverId server_id = 8
新服数据库目的地
db_host1 = 127.0.0.1 db_user1 = root db_pass1 =root #用户登录数据login db_host2 = 127.0.0.1 db_user2 = root db_pass2 = root
资源
db_host = 127.0.0.1 db_user = root db_pass = root #需要导入资源 db_list = [] ` 没怎么写过,网上查了下语法,可能不规范,大致就是这个样子的 linux一般默认版本为2.需要更新python版本到3. 需要安装pip3 命令 需要 pip3安装records pymysql 两个库 数据库信息,表信息,通过配置文件config配置完成
`#! /usr/bin/env python3 import configparser import os
import records
def init_config(): global db_app_count global db_host1 global db_user1 global db_pass1 global db_host2 global db_user2 global db_pass2 global db_host global db_user global db_pass global db_list root_dir = os.path.abspath(".") # root_dir = os.path.dirname(dir) # 获取当前文件所在目录的上一级目录，即项目所在目录E:\Crawler configpath = os.path.join(root_dir, "config.ini") cf = configparser.ConfigParser() cf.read(configpath, 'UTF-8') # 读取配置文件
db_app_count = cf.get('G2R', 'db_app_count')
db_host1 = cf.get('G2R', 'db_host1')
db_user1 = cf.get('G2R', 'db_user1')
db_pass1 = cf.get('G2R', 'db_pass1')
db_host2 = cf.get('G2R', 'db_host2')
db_user2 = cf.get('G2R', 'db_user2')
db_pass2 = cf.get('G2R', 'db_pass')
db_host = cf.get('G2R', 'db_host')
db_user = cf.get('G2R', 'db_user')
db_pass = cf.get('G2R', 'db_pass')
db_list = eval(cf.get('G2R', 'db_list'))
def connectDB(): global SOURCE_DB global APP_DB1 global APP_DB2 APP_DB1 = records.Database('mysql+pymysql://' + db_user1 + ':' + db_pass1 + '@' + db_host1 + '/gs?charset=utf8mb4') if db_app_count == 2: APP_DB2 = records.Database( 'mysql+pymysql://' + db_user2 + ':' + db_pass2 + '@' + db_host2 + '/gs?charset=utf8mb4') SOURCE_DB = records.Database('mysql+pymysql://' + db_user + ':' + db_pass + '@' + db_host + '/rs?charset=utf8mb4') print("init DB connection finish")
def scan_data_Info(): for table in db_list: workRows = APP_DB1.query("select * from " + table) insert_info(workRows, table) print("APP_DB1 scan company finish") if db_app_count == 2: workRows = APP_DB2.query("select * from " + table) insert_info(workRows, table) print("APP_DB2 scan company finish")
def insert_info(workRows, table): if len(workRows.dataset._data) > 0: param = ','.join(workRows.dataset.headers) names = ',:'.join(workRows.dataset.headers) names = ":" + names ret = SOURCE_DB.bulk_query( 'INSERT INTO ' + table + ' (' + param + ')VALUE(' + names + ')', workRows.dataset.dict); print(table) print(len(workRows))
if name == "main": try: init_config() connectDB() scan_data_Info() except Exception as e: print(e) `