-
Notifications
You must be signed in to change notification settings - Fork 2
/
backup.py
339 lines (270 loc) · 16.2 KB
/
backup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
from json.decoder import JSONDecodeError
import os
import subprocess
import platform
from datetime import datetime as date
from hardlink_info import HardlinkInfo
import info_handler
from datenbank import Blob, Datenbank, File, Jewel
from pathlib import Path
import time
class Backup:
def __init__(self, jewel_path_list, destination, testcase=False):
self.jewel_path_list = jewel_path_list
self.destination = destination
self.db = Datenbank(testcase)
self.device_name = platform.node()
if (testcase):
self.device_name = "testCases"
self.fullbackup_name = "fullBackup" + self.device_name
self.current_date_time = date.now()
self.current_date_time_formatted = self.current_date_time.strftime("%Y-%m-%d-%H-%M-%s")
self.new_backup_location = f"backup-{self.current_date_time_formatted}"
self.current_source_path = None
def initialize_backup(self, verbose_level):
start_time = time.time()
#Checks and deletes the first backup when it runned not through and start from scratch again.
if os.path.exists("db.log"):
file = open("db.log", "r")
log_lines= file.readlines()
file.close()
if len(log_lines) > 1:
if log_lines[1].rstrip() == self.fullbackup_name:
info_handler.check_db_hash(self.destination, self.fullbackup_name)
os.remove(self.destination + "/" + "tmp.db")
# to minimize work, first check if these paths even exists, then continue
tmp = self.filter_non_existing_paths(self.jewel_path_list)
diff_backup_sources = self.db.check_which_jewel_sources_exist(tmp, self.device_name)
# filter out everything, that is in diff_backup already
full_backup_sources = [e for e in tmp if e not in diff_backup_sources]
# execute,when not empty
if diff_backup_sources:
self.execute_backup(diff_backup_sources, verbose_level, start_time)
# execute, when not empty
if full_backup_sources:
self.execute_fullbackup(full_backup_sources, verbose_level, start_time)
def execute_backup(self, jewel_sources, verbose_level, start_time):
differential_backup_name = f"diff-{self.current_date_time_formatted}"
backup_sources_for_r_sync = " ".join(jewel_sources)
info_handler.check_db_hash(self.destination, differential_backup_name)
subprocess_return_verbose = self.call_rsync_differential('aAXlnvv', backup_sources_for_r_sync, differential_backup_name)
subprocess_return = self.call_rsync_differential('aAXln', backup_sources_for_r_sync, differential_backup_name)
output_array = subprocess_return.splitlines()
insert_results = self.read_files_and_jewel_from_rsync_output(output_array, jewel_sources,
f"{self.destination}/{differential_backup_name}",
self.destination + "/" + self.fullbackup_name)
self.call_rsync_differential('aAX', backup_sources_for_r_sync, differential_backup_name)
for hardlink_info in insert_results:
self.set_hardlink(hardlink_info)
info_handler.update_db_hash(self.destination, differential_backup_name)
self.print_feedback(verbose_level, differential_backup_name, 'differential', subprocess_return_verbose, start_time)
def execute_fullbackup(self, jewel_sources, verbose_level, start_time):
info_handler.check_db_hash(self.destination, self.fullbackup_name)
jewel_path_list_string = self.list_to_string(jewel_sources)
output = self.call_rsync_full('aAX', jewel_path_list_string)
output_array = output.splitlines()
insert_results = self.read_files_and_jewel_from_rsync_output(output_array, jewel_sources,
f"{self.destination}/{self.fullbackup_name}",
self.destination + "/" + self.fullbackup_name)
subprocess_return_verbose = self.call_rsync_full('aAXlnvv', jewel_path_list_string)
for hardlink_info in insert_results:
self.set_hardlink(hardlink_info)
info_handler.update_db_hash(self.destination, self.fullbackup_name)
self.print_feedback(verbose_level, self.fullbackup_name, 'full', subprocess_return_verbose, start_time)
def list_to_string(self, string_list) -> str:
formatted_string = " ".join(string_list)
return formatted_string
def filter_non_existing_paths(self, paths) -> list[str]:
return [jewel_path for jewel_path in paths if os.path.exists(jewel_path)]
def read_files_and_jewel_from_rsync_output(self, output_array, jewel_sources, store_destination_body,
fullbackup_store_destination_body) -> list[str | bool]:
result = []
if output_array == []:
exit #TODO Was macht das exit?
index = 0
for line in output_array:
# needed, since sometimes the first line ist not the jewel path,
#happens only, if subfolder content changed
if index == 0 or line.endswith('/'):
# better solution? the for is needed in both cases, but each case need another prep work
if index == 0:
self.current_source_path = os.path.dirname(line)+ "/"
if line.endswith('/'):
self.current_source_path = line
# check wether path is now the jewel
for jewel_path in jewel_sources:
# stripping and splitting is needed, since comparison does not work otherwise
if jewel_path.rsplit('/', 1)[1].strip("/") == line.strip("/"):
jewel = Jewel(0, None, date.today(), jewel_path, self.device_name,
f'{fullbackup_store_destination_body}/{line.strip("/")}')
break
# if top layer of jewel was not changed, the jewel would not be in line.strip... so we need to split and get the first folder
elif jewel_path.rsplit('/', 1)[1].strip("/") == line.split("/")[0]:
jewel = Jewel(0, None, date.today(), jewel_path, self.device_name,
f'{fullbackup_store_destination_body}/{line.strip("/")}')
break
else:
# get only the working dir without the jewel(because line inherits the jewel)
working_dir = jewel_path.rsplit('/', 1)[0]
file_object = info_handler.get_metadata(working_dir + '/' + line)
# Erstellt Array erstes element vor letztem Slash, zweites Element nach dem Slash
file_name = line.rsplit('/', 1)[1]
blob = Blob(0, 0, file_object.f_hash, (f'{self.db._encode_base64(file_name)}_{file_object.f_hash}'),
file_object.f_size,
self.current_date_time, file_object.modify, 0, file_name,
working_dir + "/" + line, f'{store_destination_body}/{line}')
file = File(0, [blob], file_object.birth)
db_answer = self.db.add_to_database(jewel, file, self.device_name)
if db_answer is not True:
result.append(HardlinkInfo(db_answer[0].id, db_answer[0].store_destination, blob.store_destination, self.current_date_time,blob.origin_name, working_dir + "/" + line, jewel.id, db_answer[1]))
index = index +1
return result
# Description: Generate a String of excluded files which are handed over ,to the rsync command.
# Input: None
# Output: String containing the rsync option "--exclude" and the name of the excluded
# files. Example: "--exclude "file1" --exclude "file2" ... --exclude "fileN""
def excluding_data(self):
config = info_handler.get_json_info(self.device_name)
return_list = []
for element in config['blacklist']['directories'] + config['blacklist']['files']:
return_list.append(f'--exclude "{element}"')
for extension in config['blacklist']['extensions']:
return_list.append(f'--exclude *"{extension}"')
return ' '.join(return_list)
def set_hardlink(self, hl_info:HardlinkInfo ):
d_path = os.path.dirname(os.path.abspath(hl_info.destination_path))
try:
os.listdir(d_path)
#subprocess.run(f'ls "{d_path}"', shell=True)
except:
print(f'Error: ls couldn\'t be executed\n'
f'path: {d_path}\n')
exit()
os.remove(hl_info.destination_path)
try:
os.link(hl_info.link_path,hl_info.destination_path)
#subprocess.run(f'ln "{hl_info.link_path}" "{hl_info.destination_path}"', shell=True)
except:
print(f'Error: ln couldn\'t be executed\n'
f'origin path: {hl_info.link_path}\n'
f'destination path: {hl_info.destination_path}\n')
exit()
if not hl_info.old_hardlink_existing:
self.db.protocol_hardlink(hl_info, self.device_name)
def print_feedback(self, verbose_level: int, backup_name: str, backup_type: str, subprocess_return_verbose: str, start_time: time):
if backup_type == 'full':
backup_path = f'{self.destination}/{self.fullbackup_name}'
else:
backup_path = f'{self.destination}/{backup_name}'
files_amount = sum([len(files) for r, d, files in os.walk(backup_path)])
files_size = sum(f.stat().st_size for f in Path(backup_path).glob('**/*') if f.is_file())
backup_version = len(next(os.walk(self.destination))[1])
config = info_handler.get_json_info(self.device_name)
input_paths = config['jewel_sources'][self.device_name]
excluded_amount = 0
excluded_size = 0
for line in subprocess_return_verbose.splitlines():
if '[sender] hiding file' in line:
excluded_file = line.split(' ')[3].split('/')[-1]
excluded_amount += 1
for input_path in reversed(input_paths):
try:
excluded_size += os.stat(f'{input_path}/{excluded_file}').st_size
except:
pass
elif '[sender] hiding directory' in line:
excluded_directory = ''.join(line.split(' ')[3].split('/')[1:])
for input_path in reversed(input_paths):
try:
excluded_amount += sum([len(files) for r, d, files in os.walk(f'{input_path}/{excluded_directory}')])
except:
pass
for input_path in reversed(input_paths):
try:
for path, dirs, files in os.walk(f'{input_path}/{excluded_directory}'):
for f in files:
fp = os.path.join(path, f)
excluded_size += os.path.getsize(fp)
except:
pass
total_amount = sum([len(files) for r, d, files in os.walk(self.destination) if 'datenbank.db' not in files])
total_size = sum(f.stat().st_size for f in Path(self.destination).glob('**/*') if f.is_file() and f.name != 'datenbank.db')
database_size = os.stat(f'{self.destination}/datenbank.db').st_size
files_size_unit = 'bytes'
excluded_size_unit = 'bytes'
total_size_unit = 'bytes'
database_size_unit = 'bytes'
unit_list = ['bytes', 'kilobytes', 'megabytes', 'gigabytes']
for i in range(len(unit_list) - 1, 0, -1):
if (files_size / pow(1000, i)) >= 1:
files_size = round((files_size / pow(1000, i)), 2)
files_size_unit = unit_list[i]
if (excluded_size / pow(1000, i)) >= 1:
excluded_size = round((excluded_size / pow(1000, i)), 2)
excluded_size_unit = unit_list[i]
if (total_size / pow(1000, i)) >= 1:
total_size = round((total_size / pow(1000, i)), 2)
total_size_unit = unit_list[i]
if (database_size / pow(1000, i)) >= 1:
database_size = round((database_size / pow(1000, i)), 2)
database_size_unit = unit_list[i]
if verbose_level == 0:
print(f'\nCURRENT BACKUP DETAILS\n'
f'├── backup name:\t\t\t{backup_name}\n'
f'├── number of files:\t\t\t{files_amount}\n'
f'│ └── size of files:\t\t\t{files_size} {files_size_unit}\n'
f'└── time elapsed:\t\t\t{round((time.time() - start_time), 2)} seconds\n')
if backup_type == 'differential':
print(f'\nOVERALL BACKUP DETAILS\n'
f'└── number of files:\t\t\t{total_amount}\n'
f' └── size of files:\t\t\t{total_size} {total_size_unit}\n')
elif verbose_level == 1:
print(f'\nCURRENT BACKUP DETAILS\n'
f'├── backup name:\t\t\t{backup_name}\n'
f'├── backup type:\t\t\t{backup_type}\n'
f'├── backup version:\t\t\t{backup_version}\n'
f'├── number of files:\t\t\t{files_amount}\n'
f'│ └── size of files:\t\t\t{files_size} {files_size_unit}\n'
f'├── number of exluded files:\t\t{excluded_amount}\n'
f'│ └── size of excluded files:\t\t{excluded_size} {excluded_size_unit}\n'
f'└── time elapsed:\t\t\t{round((time.time() - start_time), 2)} seconds\n')
if backup_type == 'differential':
print(f'\nOVERALL BACKUP DETAILS\n'
f'├── number of files:\t\t\t{total_amount}\n'
f'│ └── size of files:\t\t\t{total_size} {total_size_unit}\n'
f'└── size of database:\t\t\t{database_size} {database_size_unit}\n')
def call_rsync_differential(self, options: str, backup_sources_for_r_sync: str, backup_name: str):
try:
subprocess_output = subprocess.Popen(f"rsync -{options} {self.excluding_data()} --out-format='%n' "
f"--compare-dest={self.destination}/{self.fullbackup_name} {backup_sources_for_r_sync} "
f"{self.destination}/{backup_name}",
shell=True,
stdout=subprocess.PIPE)
return self.wait_decode_subprocess(subprocess_output)
except:
print(f'Error: rsync couldn\'t be executed\n'
f'used option: {options}\n'
f'fullbackup path: {self.destination}/{self.fullbackup_name}\n'
f'destination path: {self.destination}/{backup_name}\n')
exit()
def call_rsync_full(self, options: str, jewel_path_list_string: str):
try:
subprocess_output = subprocess.Popen(f'rsync -{options} {self.excluding_data()} --out-format="%n" '
f'{jewel_path_list_string} '
f'{self.destination}/{self.fullbackup_name}',
shell=True,
stdout=subprocess.PIPE)
return self.wait_decode_subprocess(subprocess_output)
except:
print(f'Error: rsync couldn\'t be executed\n'
f'used option: {options}\n'
f'jewel path: {jewel_path_list_string}\n'
f'destination path: {self.destination}/{self.fullbackup_name}\n')
exit()
def wait_decode_subprocess(self, subprocess_output):
output=[]
output = [line for line in subprocess_output.stdout]
return_value=''.join([x.decode('utf-8') for x in output])
subprocess_output.stdout.close()
subprocess_output.wait()
return return_value