From 43f6df04a6eb636fa591ac57ae5d7a429017a556 Mon Sep 17 00:00:00 2001 From: "black.box (Unzoner) team@belodetek.io" Date: Fri, 1 Dec 2023 15:09:41 -0800 Subject: [PATCH] fix: reset loop if any tunnel interface is not passing traffic --- unzoner/.balena/balena.yml | 2 +- unzoner/src/main.py | 288 ++++++++++++++++--------------------- 2 files changed, 122 insertions(+), 168 deletions(-) diff --git a/unzoner/.balena/balena.yml b/unzoner/.balena/balena.yml index 8e8bba7..4feb25e 100644 --- a/unzoner/.balena/balena.yml +++ b/unzoner/.balena/balena.yml @@ -22,7 +22,7 @@ build-variables: - BUILD_WANPROXY_VERSION=0.8.0 - BUILD_WANPROXY=0 # https://nuitka.net/ - - COMPILE_CODE=1 + - COMPILE_CODE=0 # (e.g.) dig +short us.{{ DNS_SUB_DOMAIN }}.{{ DNS_DOMAIN }} - DNS_SUB_DOMAIN=blackbox diff --git a/unzoner/src/main.py b/unzoner/src/main.py index cdbce1f..b4f4cca 100755 --- a/unzoner/src/main.py +++ b/unzoner/src/main.py @@ -80,11 +80,7 @@ def main(): '^.*ifconfig ([\w\d]+) ([\d]+\.[\d]+\.[\d]+\.[\d]+) pointopoint ([\d]+\.[\d]+\.[\d]+\.[\d]+) mtu [\d]+$' ) - if DEVICE_TYPE == 5: - p2 = re.compile( - '^.*get_ping_host: route_network_[0-9]+=([\d]+\.[\d]+\.[\d]+\.[\d]+)$' - ) - p3 = re.compile('^.*remote=(.*) country=(.*)$') + p3 = re.compile('^.*remote=(.*) country=(.*)$') # hdparm tests p4 = re.compile( @@ -96,6 +92,10 @@ def main(): '^(\d+\s+.*)\s+\((.*),.*\)\s+.*,\s+(.*),\s+(.*)$' ) + p6 = re.compile( + '^.*get_ping_host: route_network_[0-9]+=([\d]+\.[\d]+\.[\d]+\.[\d]+)$' + ) + mgmt_ipaddr = None if TUN_MGMT: mgmt_ipaddr = get_ip_address(MGMT_IFACE) @@ -156,30 +156,26 @@ def main(): dev = m2[0] s_local[idx] = m2[1] s_peer = m2[2] - log( - '{}: iface={} local={} peer={} proto={}'.format( - stack()[0][3], - dev, - s_local[idx], - s_peer, - proto - ) - ) + log('{}: iface={} s_local={} s_peer={} proto={}'.format( + stack()[0][3], + dev, + s_local[idx], + s_peer, + proto + )) except (IndexError, TypeError, AttributeError): pass if not started[idx] and starting[idx]: if i % LOOP_CYCLE == 0 and not started[idx]: - log( - '{}: cycle={} started={} starting={} proto={} s_pid={}'.format( - stack()[0][3], - i, - started[idx], - starting[idx], - proto, - s_pid[idx] - ) - ) + log('{}: cycle={} started={} starting={} proto={} s_pid={}'.format( + stack()[0][3], + i, + started[idx], + starting[idx], + proto, + s_pid[idx] + )) started[idx] = False starting[idx] = False s_proc[idx].terminate() @@ -188,14 +184,12 @@ def main(): if not started[idx] and not starting[idx]: if i == 1: # once per loop (start) starting[idx] = True - log( - '{}: cycle={} starting={} proto={}'.format( - stack()[0][3], - i, - starting[idx], - proto - ) - ) + log('{}: cycle={} starting={} proto={}'.format( + stack()[0][3], + i, + starting[idx], + proto + )) try: (s_stdoutq[idx], s_stderrq[idx], s_proc[idx]) = start_server( @@ -215,13 +209,11 @@ def main(): ) kill_thread.daemon = True kill_thread.start() - log( - 'disconnect_clients: name={} alive={} daemon={}'.format( - kill_thread.name, - kill_thread.is_alive(), - kill_thread.daemon - ) - ) + log('disconnect_clients: name={} alive={} daemon={}'.format( + kill_thread.name, + kill_thread.is_alive(), + kill_thread.daemon + )) if not auth_thread.is_alive(): if BITCOIN_PAYMENT_CHECK: @@ -231,22 +223,18 @@ def main(): ) auth_thread.daemon = True auth_thread.start() - log( - 'reauthenticate_clients: name={} alive={} daemon={}'.format( - auth_thread.name, - auth_thread.is_alive(), - auth_thread.daemon - ) - ) + log('reauthenticate_clients: name={} alive={} daemon={}'.format( + auth_thread.name, + auth_thread.is_alive(), + auth_thread.daemon + )) if i == 1 or i % LOOP_CYCLE == 0: # twice per loop (start and end) - if DEBUG: log( - 'get_status: cycle={} proto={} status={}'.format( - i, - proto, - get_status(proto=proto) - ) - ) + if DEBUG: log('get_status: cycle={} proto={} status={}'.format( + i, + proto, + get_status(proto=proto) + )) s_conns[idx] = int( get_server_conns_by( proto=proto @@ -260,15 +248,13 @@ def main(): except AssertionError as e: print(repr(e)) if DEBUG: print_exc() - log( - '{}: cycle={} proto={} s_local={} s_loss={}'.format( - i, - stack()[0][3], - proto, - s_local[idx], - s_loss[idx] - ) - ) + log('{}: cycle={} proto={} s_local={} s_loss={}'.format( + i, + stack()[0][3], + proto, + s_local[idx], + s_loss[idx] + )) started[idx] = False starting[idx] = False s_proc[idx].terminate() @@ -286,20 +272,18 @@ def main(): shell_check_output_cmd( 'ip link | grep %s' % TUN_IFACE ) - gwip = shell_check_output_cmd( + c_gwip = shell_check_output_cmd( "ip route | grep %s | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ via [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ dev %s' | awk '{print $1}'" % ( TUN_IFACE, TUN_IFACE ) ) - log( - 'iface={} gwip={}'.format( - TUN_IFACE, - gwip - ) - ) + log('iface={} c_gwip={}'.format( + TUN_IFACE, + c_gwip + )) ping_host( - host=gwip.strip('\n'), + host=c_gwip.strip('\n'), timeout=1, count=3 ) @@ -310,11 +294,9 @@ def main(): if DEBUG: print_exc() except: if DEBUG: print_exc() - log( - '{}: double-vpn not ready'.format( - stack()[0][3] - ) - ) + log('{}: double-vpn not ready'.format( + stack()[0][3] + )) try: # force server down if double-vpn is down log_server_stats() @@ -348,13 +330,11 @@ def main(): if DEVICE_TYPE in [2, 3, 5]: # iperf timeout if c_stp and now - c_stimer > (LOOP_TIMER * LOOP_CYCLE * 5): - log( - '{}: pid={} elapsed={}'.format( - stack()[0][3], - c_stp.pid, - now - c_stimer - ) - ) + log('{}: pid={} elapsed={}'.format( + stack()[0][3], + c_stp.pid, + now - c_stimer + )) c_stp.terminate() c_stp = None c_stq = None @@ -376,14 +356,12 @@ def main(): # hdparm or dd timeout if c_iotp and now - c_iotimer > (LOOP_TIMER * LOOP_CYCLE * 3): - log( - '{}: pid={} elapsed={} test={}'.format( - stack()[0][3], - c_iotp.pid, - now - c_iotimer, - c_iott - ) - ) + log('{}: pid={} elapsed={} test={}'.format( + stack()[0][3], + c_iotp.pid, + now - c_iotimer, + c_iott + )) try: result = { 'status': 1, @@ -392,12 +370,10 @@ def main(): } log('run_iotest: result={}'.format(result)) res = update_iotest(data=result) - log( - 'update_iotest({}): {}'.format( - res[0], - res[1] - ) - ) + log('update_iotest({}): {}'.format( + res[0], + res[1] + )) except Exception as e: print(repr(e)) if DEBUG: print_exc() @@ -484,12 +460,10 @@ def main(): log('run_iotest: result={}'.format(result)) try: res = update_iotest(data=result) - log( - 'update_iotest({}): {}'.format( - res[0], - res[1] - ) - ) + log('update_iotest({}): {}'.format( + res[0], + res[1] + )) except Exception as e: print(repr(e)) if DEBUG: print_exc() @@ -530,24 +504,15 @@ def main(): ) except: if DEBUG: print_exc() + try: - m2 = p2.search(c_msg).groups() - dev = m2[0] - c_local = m2[1] - c_peer = m2[2] - c_gwip = c_peer.split('.') - c_gwip[3] = '1' - c_gwip = '.'.join(c_gwip) - log( - '{}: iface={} local={} peer={} gwip={} family={}'.format( - stack()[0][3], - dev, - c_local, - c_peer, - c_gwip, - AF - ) - ) + m6 = p6.search(c_msg).groups() + c_gwip = m6[0] + log('{}: c_gwip={} family={}'.format( + stack()[0][3], + c_gwip, + AF + )) except (IndexError, TypeError, AttributeError): pass @@ -556,27 +521,23 @@ def main(): m3 = p3.search(c_msg).groups() c_remote = m3[0] c_country = m3[1] - log( - '{}: remote={} country={}'.format( - stack()[0][3], - c_remote, - c_country - ) - ) + log('{}: remote={} country={}'.format( + stack()[0][3], + c_remote, + c_country + )) except (IndexError, TypeError, AttributeError): pass if not connected and connecting: if i % LOOP_CYCLE == 0 and not connected: - log( - '{}: cycle={} connected={} connecting={} c_pid={}'.format( - stack()[0][3], - i, - connected, - connecting, - c_pid - ) - ) + log('{}: cycle={} connected={} connecting={} c_pid={}'.format( + stack()[0][3], + i, + connected, + connecting, + c_pid + )) connected = False connecting = False c_proc.terminate() @@ -585,14 +546,12 @@ def main(): if not connected and not connecting: if i == 1: connecting = True - log( - '{}: cycle={} connecting={} family={}'.format( - stack()[0][3], - i, - connecting, - AF - ) - ) + log('{}: cycle={} connecting={} family={}'.format( + stack()[0][3], + i, + connecting, + AF + )) try: (c_stdoutq, c_stderrq, c_proc, c_proto) = connect_node( family=AF @@ -615,13 +574,11 @@ def main(): print(repr(e)) if DEBUG: print_exc() if c_stimer: - log( - 'run_speedtest: pid={} elapsed={} results={}'.format( - c_stp.pid, - now - c_stimer, - len(c_str) - ) - ) + log('run_speedtest: pid={} elapsed={} results={}'.format( + c_stp.pid, + now - c_stimer, + len(c_str) + )) # run hdparm or dd try: @@ -638,29 +595,26 @@ def main(): print(repr(e)) if DEBUG: print_exc() if c_iotimer: - log( - 'run_iotest: pid={} elapsed={} test={} result={}'.format( - c_iotp.pid, - now - c_iotimer, - c_iott, - c_iotl - ) - ) - - if i % POLL_FREQ == 0 and c_gwip: + log('run_iotest: pid={} elapsed={} test={} result={}'.format( + c_iotp.pid, + now - c_iotimer, + c_iott, + c_iotl + )) + + # check if client side tunnel is still passing traffic (assume not) + if i % POLL_FREQ == 0 and c_gwip: # every x cycles per loop c_loss = 100 try: c_loss = ping_host(host=c_gwip) except AssertionError as e: print(repr(e)) if DEBUG: print_exc() - log( - '{}: c_gwip={} c_loss={}'.format( - stack()[0][3], - c_gwip, - c_loss - ) - ) + log('{}: c_gwip={} c_loss={}'.format( + stack()[0][3], + c_gwip, + c_loss + )) connected = False connecting = False c_proc.terminate()