-
Notifications
You must be signed in to change notification settings - Fork 0
/
rbdbigbackup.py
262 lines (240 loc) · 11.2 KB
/
rbdbigbackup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import sys
import commands
import json
import sqlite3
import time
def main():
if len(sys.argv) == 1 or len(sys.argv) == 2:
sys.argv.append("-h")
if sys.argv[1] == '-h' or sys.argv[1] == 'help':
help()
if len(sys.argv) == 3 :
if sys.argv[2] == 'init':
checkoutput=checkpoolimage(sys.argv[1])
if checkoutput[4] ==1:
poolname=checkoutput[0]
imagename=checkoutput[1]
prifixname=checkoutput[2].split('.',1)[1]
objects=checkoutput[3]
init_image_localdb(poolname,imagename,prifixname,objects)
else:
print "check your pool/image name!"
elif sys.argv[2] == 'update':
checkoutput=checkpoolimage(sys.argv[1])
if checkoutput[4] ==1:
poolname=checkoutput[0]
imagename=checkoutput[1]
prifixname=checkoutput[2].split('.',1)[1]
objects=checkoutput[3]
update_image_localdb(poolname,imagename,prifixname,objects)
elif sys.argv[2] == 'get':
checkoutput=checkpoolimage(sys.argv[1])
if checkoutput[4] ==1:
poolname=checkoutput[0]
imagename=checkoutput[1]
prifixname=checkoutput[2].split('.',1)[1]
objects=checkoutput[3]
get_image_localdb(poolname,imagename,prifixname,objects)
elif sys.argv[2] == 'fstrim':
checkoutput=checkpoolimage(sys.argv[1])
if checkoutput[4] ==1:
poolname=checkoutput[0]
imagename=checkoutput[1]
prifixname=checkoutput[2].split('.',1)[1]
objects=checkoutput[3]
fstrim_image_localdb(poolname,imagename,prifixname,objects)
elif sys.argv[2] == 'build':
prifixname=sys.argv[1]
checkdbifexit(prifixname)
build_image_localdb(prifixname)
else:
help()
def processshow(num,objects):
print "当前的进度:{0}/{1} {2}%\r".format(num,objects,round((num + 1) * 100 / objects)),
time.sleep(0.01)
#函数判断为poolname/imagename组合的形式,判断包含字符串，然后以字符串为分割得到存储池名称和image名称，否则说没有
def checkpoolimage(name):
if name.count("/") == 1:
if len(name.split('/',1)[0])!=0 and len(name.split('/',1)[1])!=0:
poolname = name.split('/',1)[0]
imagename = name.split('/',1)[1]
rbd_info = commands.getoutput('rbd info %s/%s --format json --pretty-format 2>/dev/null'%(poolname,imagename) )
try:
json_str=json.loads(rbd_info)
rbd_prifix=json_str['block_name_prefix']
objects=json_str['objects']
return [poolname,imagename,rbd_prifix,objects,1]
except:
return ['','','','',0]
else:
return ['','','','',0]
else:
return ['','','','',0]
#需要判断下本地数据库是否记录了这个rbd 的相关信息（正常应该有的）
def checkdbifexit(prifixname):
conn = sqlite3.connect('rbd.db')
c = conn.cursor()
#初始化数据库的表
def init_image_localdb(poolname,imagename,prifixname,objects):
conn = sqlite3.connect('rbd.db')
c = conn.cursor()
try:
c.execute("drop table [%s-%s-%s-%s]" %(poolname,imagename,prifixname,objects))
except:
pass
c.execute("CREATE TABLE [%s-%s-%s-%s](objectnum PRIMARY KEY,objectname TEXT,getmtime,savemtime,ifget)" %(poolname,imagename,prifixname,objects))
print "初始化本地数据库:"
for num in range(objects):
num_to_hex = commands.getoutput('printf "%.16x\n"' %num )
c.execute("INSERT INTO [%s-%s-%s-%s](objectnum,objectname,getmtime,savemtime,ifget) VALUES(%s,'rbd_data.%s.%s',NULL,NULL,0)" %(poolname,imagename,prifixname,objects,num,prifixname,num_to_hex))
#####计时器技术
processshow(num,objects)
####
conn.commit()
print ""
print("本地数据库rbd.db生成数据库表: %s-%s-%s-%s" %(poolname,imagename,str(prifixname),objects) )
conn.close()
def update_image_localdb(poolname,imagename,prifixname,objects):
conn = sqlite3.connect('rbd.db')
c = conn.cursor()
print "更新本地数据库:"
search=0
update=0
for num in range(objects):
num_to_hex = commands.getoutput('printf "%.16x\n"' %num )
getmtime=commands.getstatusoutput('rados stat -p %s rbd_data.%s.%s 2>/dev/null' %(poolname,prifixname,num_to_hex))
if getmtime[0] == 0:
search=search+1
newgetmtime=getmtime[1].split(' ',-1)[2]+" "+getmtime[1].split(' ',-1)[3]
c.execute("UPDATE [%s-%s-%s-%s] SET getmtime = '%s' WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,newgetmtime,num))
elif getmtime[0] != 0:
c.execute("UPDATE [%s-%s-%s-%s] SET getmtime = NULL,ifget = '0' WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,num))
cursor=c.execute("SELECT getmtime,savemtime FROM [%s-%s-%s-%s] WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,num))
for row in cursor:
if row[0] != row[1] and row[0] != None:
c.execute("UPDATE [%s-%s-%s-%s] SET ifget = '1' WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,num))
update=update+1
conn.commit()
processshow(num,objects)
print ""
print("更新数据库rbd.db数据库表: %s-%s-%s-%s,本次查询对象数:%s,下次需要下载对象数:%s" %(poolname,imagename,str(prifixname),objects,search,update))
conn.close()
def get_image_localdb(poolname,imagename,prifixname,objects):
conn = sqlite3.connect('rbd.db')
c = conn.cursor()
try:
os.makedirs("%s/%s" %(poolname,imagename))
except:
pass
print "下载RBD的对象:"
downid=1
checkdown=c.execute("SELECT COUNT(*) FROM [%s-%s-%s-%s] WHERE ifget=\"1\" ;" %(poolname,imagename,prifixname,objects) )
for num in checkdown:
print "本次需要下载对象数目:",num[0]
needdown=num[0]
for num in range(objects):
num_to_hex = commands.getoutput('printf "%.16x\n"' %num )
cursor=c.execute("SELECT ifget FROM [%s-%s-%s-%s] WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,num) )
for row in cursor:
if row[0] == '1':
savemtime=commands.getstatusoutput('rados stat -p %s rbd_data.%s.%s |awk \'{print $3,$4}\' 2>/dev/null'%(poolname,prifixname,num_to_hex))
getobject=commands.getstatusoutput('rados -p %s get rbd_data.%s.%s %s/%s/rbd_data.%s.%s 2>/dev/null'%(poolname,prifixname,num_to_hex,poolname,imagename,prifixname,num_to_hex))
if savemtime[0] == 0 and getobject[0] == 0:
c.execute("UPDATE [%s-%s-%s-%s] SET ifget = '0',savemtime = '%s' WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,savemtime[1],num) )
processshow(downid,needdown)
downid=downid+1
conn.commit()
print ""
print poolname,imagename,"对象下载完成"
conn.close()
def fstrim_image_localdb(poolname,imagename,prifixname,objects):
conn = sqlite3.connect('rbd.db')
c = conn.cursor()
try:
os.makedirs("%s/%s/fstrim" %(poolname,imagename))
except:
pass
for num in range(objects):
num_to_hex = commands.getoutput('printf "%.16x\n"' %num )
cursor=c.execute("SELECT getmtime FROM [%s-%s-%s-%s] WHERE objectnum = %s ;" %(poolname,imagename,prifixname,objects,num) )
for row in cursor:
if row[0] == None:
mvfile=commands.getoutput('mv %s/%s/rbd_data.%s.%s %s/%s/fstrim/ 2>/dev/null'%(poolname,imagename,prifixname,num_to_hex,poolname,imagename))
conn.commit()
print ""
print poolname,imagename,"清理完成"
conn.close()
def build_image_localdb(prifixname):
conn = sqlite3.connect('rbd.db')
c = conn.cursor()
cursor=c.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
print "拼接RBD块设备:"
for row in cursor:
dbprifixname=row[0].split('-',-1)[2]
if dbprifixname == prifixname:
dbpoolname=row[0].split('-',-1)[0]
dbimagename=row[0].split('-',-1)[1]
dbobjects=row[0].split('-',-1)[3]
obj_size=4194304
rebuild_block_size=512
rbd_size=int(obj_size)*int(dbobjects)
try:
mklocalfile=commands.getoutput('dd if=/dev/zero of=%s/%s/%s bs=1 count=0 seek=%d 2>/dev/null'%(dbpoolname,dbimagename,dbimagename,rbd_size))
except:
print "检查本地的rbd路径对象是否存在"
#print buildfile
files = [f for f in os.listdir('./%s/%s/' %(dbpoolname,dbimagename)) if os.path.isfile("./%s/%s/%s" %(dbpoolname,dbimagename,f)) and prifixname in f]
filenum=len(files)
for index,f in enumerate(files):
getseek_loc=commands.getoutput('echo %s | awk -F_ \'{print $2}\' | awk -v os=%d -v rs=%d -F. \'{print os*strtonum("0x" $NF)/rs}\'' %(f,obj_size,rebuild_block_size) )
dd_to_file=commands.getoutput( 'dd conv=notrunc if=%s/%s/%s of=%s/%s/%s seek=%s bs=%s 2>/dev/null' %(dbpoolname,dbimagename,f,dbpoolname,dbimagename,dbimagename,getseek_loc,rebuild_block_size))
processshow(index,filenum)
print ""
print "生成本地Image:",dbimagename
def help(): print """Usage : rbdbigbackup.py [-h] [poolname/imagename|prifixname] [command]
Ceph export tools - Version 1.0
OPTIONS
========
-h Print help
COMMANDS
=========
--------
|\033[0;32;40m init\033[0m |
--------
[poolname/imagename] init 初始化本地数据库（重新全量备份的时候也可以执行）
--------
|\033[0;32;40m update\033[0m |
--------
[poolname/imagename] update 更新数据库中需要增量备份的对象
--------
|\033[0;32;40m get\033[0m |
--------
[poolname/imagename] get 下载增量的部分
--------
|\033[0;32;40m fstrim\033[0m |
--------
[poolname/imagename] fstrim 清理远端已经删除的对象到目录fstrim
--------
|\033[0;32;40m build\033[0m |
--------
[prifixname] build 根据本地的对象的prifix对数据进行拼接
--------
-----------------------------------------------------------------------
操作步骤：
[备份过程]
init 来初始化本地数据库 ---> update 定期更新记录需要备份的对象 ---> get 下载数据库中记录需要更新的数据 ---> fstrim 清理本地的垃圾数据
[恢复过程]
build是恢复数据的时候使用的,testrbd为生成的本地image:
#losetup /dev/loop0 testrbd
#mount /dev/loop0 /mnt
#losetup -d /dev/loop0
----------------------------------------------------------------
语法例子:
#init# rbdbigbackup.py rbd/testrbd init
#build# rbdbigbackup.py 103b6b8b4567 build
"""
if __name__ == '__main__':
main()