-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcheck_netxio_errors.py
144 lines (100 loc) · 4.24 KB
/
check_netxio_errors.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
'''Check NetXIO lostandfound and sync.log errors'''
import hashlib
import logging
import os
import re
import shutil
import sys
# from fileinput import filename
from utils import setup
from utils import netx_api
def list_files(directory:str):
'''get a list of files with full filepaths in a directory (recursive)'''
paths = []
for root, dirs, files in os.walk(directory):
for file in files:
file_name = file
file_path = os.path.join(root, file)
file_row = {
'name':file_name,
'path':file_path
}
paths.append(file_row)
return paths
def main(live_or_test:str=None):
'''Main function'''
# setup.start_log_dams_netx()
# if live_or_test is None:
# live_or_test = setup.get_sys_argv(number_of_args=1)
# Start logs
if live_or_test is None:
input_args = sys.argv
live_or_test = setup.get_sys_argv(number_of_args=1)
else:
input_args = [live_or_test]
setup.start_log_dams_netx(config=None, cmd_args=input_args)
config = setup.get_config_dams_netx(live_or_test)
if live_or_test == 'LIVE':
netxio_source_dir = config['NETXIO_LOCAL_SOURCE_DIR']
netxio_lost_dir = config['NETXIO_LOSTANDFOUND']
else:
netxio_source_dir = config['TEST_NETXIO_LOCAL_SOURCE_DIR']
netxio_lost_dir = config['TEST_NETXIO_LOSTANDFOUND']
lost_and_found_files = list_files(netxio_lost_dir)
# check for trailing slash in NetXIO source dir value
if re.match(r'.*/$', netxio_source_dir) is None:
netxio_source_dir += '/'
# Check each lost-&-found file
for file_row in lost_and_found_files:
file_name = file_row['name']
file_path = file_row['path']
# - by checksum
# - - get md5 of lost-found-file
with open(file_path,'rb') as media_file:
file_bytes = media_file.read()
file_md5 = hashlib.md5(file_bytes).hexdigest()
asset_data_md5 = netx_api.netx_get_asset_by_field(
search_field="fileChecksum",
search_value=file_md5,
netx_test=live_or_test
)
# Get NetX Asset ID by md5
# If found, remove
if 'result' in asset_data_md5 and len(asset_data_md5['result']['results']) > 0:
asset_id = f"NetX asset {asset_data_md5['result']['results'][0]['id']}"
log_msg = f'Removed {file_name} from lostandfound: md5 {file_md5} matches {asset_id}'
print(log_msg)
logging.info(log_msg)
os.remove(file_path)
# If not found, check by filename (next if/else)
else:
# Check if file is in NetX
# - by filename
asset_data = netx_api.netx_get_asset_by_filename(
file_name=file_name,
netx_env=live_or_test
)
if 'result' in asset_data and len(asset_data['result']['results']) > 0:
asset_id = f"NetX asset {asset_data['result']['results'][0]['id']}"
os.remove(file_path)
log_message = f'Removed {file_name} from lostandfound - guid matches {asset_id}'
print(log_message)
logging.info(log_message)
elif 'result' in asset_data and len(asset_data['result']['results']) < 1:
file_path_edit = re.sub(netxio_lost_dir ,'', file_path)
file_path_edit = re.sub(file_name ,'', file_path_edit)
moving_file = f'file for NetXIO returned to: {netxio_source_dir + file_path_edit}'
log_message = f'No NetX match for {file_name} (md5 {file_md5}) - {moving_file}'
print(log_message)
logging.info(log_message)
if os.path.exists(netxio_source_dir + file_path_edit) is False:
os.makedirs(netxio_source_dir + file_path_edit)
if os.path.exists(netxio_source_dir + file_path_edit + file_name) is False:
shutil.move(file_path, netxio_source_dir + file_path_edit)
else:
log_message = 'Possible API error'
print(log_message)
logging.error(log_message)
setup.stop_log_dams_netx()
if __name__ == '__main__':
main()